Merge pull request #47188 from caseydavenport/calico-typha

Automatic merge from submit-queue (batch tested with PRs 47000, 47188, 47094, 47323, 47124)

Add Calico typha agent

**What this PR does / why we need it**:

- Adds the Calico typha agent with autoscaling to the GCE scripts. 
- Adds logic to adjust Calico resource requests based on cluster size.

Fixes https://github.com/kubernetes/kubernetes/issues/47269

**Special notes for your reviewer**:

CC @dnardo 

**Release note**:
```release-note
NONE
```

cluster/addons/calico-policy-controller/README.md

@@ -1,14 +1,11 @@
 # Calico Policy Controller
 ==============
 
-Calico is an implementation of the Kubernetes network policy API.  The provided manifest installs a DaemonSet which runs Calico on each node in the cluster.
+Calico is an implementation of the Kubernetes network policy API.  The provided manifests install:
 
-### Templating
+- A DaemonSet which runs Calico on each node in the cluster.
-
+- A Deployment which installs the Calico Typha agent.
-The provided `calico-node.yaml` manifest includes the following placeholders which are populated
+- A Service for the Calico Typha agent.
-via templating.
-
-- `__CLUSTER_CIDR__`: The IP range from which Pod IP addresses are assigned.
 
 ### Learn More
 

cluster/addons/calico-policy-controller/calico-node-daemonset.yaml

@@ -11,6 +11,8 @@ spec:
   selector:
     matchLabels:
       k8s-app: calico-node
+  updateStrategy:
+    type: RollingUpdate
   template:
     metadata:
       labels:
@@ -26,18 +28,16 @@ spec:
         # container programs network policy and routes on each
         # host.
         - name: calico-node
-          image: calico/node:v1.2.1
+          image: calico/node:v1.3.0
           env:
             - name: CALICO_DISABLE_FILE_LOGGING
               value: "true"
             - name: CALICO_NETWORKING_BACKEND
               value: "none"
-            - name: CALICO_IPV4POOL_CIDR
-              value: "__CLUSTER_CIDR__"
-            - name: CALICO_IPV4POOL_IPIP
-              value: "off"
             - name: DATASTORE_TYPE
               value: "kubernetes"
+            - name: FELIX_TYPHAK8SSERVICENAME
+              value: "calico-typha"
             - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
               value: "ACCEPT"
             - name: FELIX_IPV6SUPPORT
@@ -48,6 +48,8 @@ spec:
               value: "true"
             - name: IP
               value: ""
+            - name: NO_DEFAULT_POOLS
+              value: "true"
             - name: NODENAME
               valueFrom:
                 fieldRef:
@@ -58,7 +60,7 @@ spec:
             privileged: true
           resources:
             requests:
-              cpu: 250m
+              cpu: __CALICO_NODE_CPU__
           volumeMounts:
             - mountPath: /lib/modules
               name: lib-modules
@@ -69,7 +71,7 @@ spec:
         # This container installs the Calico CNI binaries
         # and CNI network config file on each node.
         - name: install-cni
-          image: calico/cni:v1.8.3-hostport
+          image: calico/cni:v1.9.1
           command: ["/install-cni.sh"]
           env:
             - name: CNI_CONF_NAME
@@ -126,11 +128,11 @@ spec:
         # Used to install CNI binaries.
         - name: cni-bin-dir
           hostPath:
-            path: /home/kubernetes/bin
+            path: __CALICO_CNI_DIR__
         # Used to install CNI network config.
         - name: cni-net-dir
           hostPath:
             path: /etc/cni/net.d
       tolerations:
-      - key: "CriticalAddonsOnly"
+        - key: "CriticalAddonsOnly"
-        operator: "Exists"
+          operator: "Exists"

cluster/addons/calico-policy-controller/typha-deployment.yaml

@@ -0,0 +1,60 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: calico-typha
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    k8s-app: calico-typha
+spec:
+  replicas: __CALICO_TYPHA_REPLICAS__
+  revisionHistoryLimit: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: calico-typha
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      nodeSelector:
+        projectcalico.org/ds-ready: "true"
+      hostNetwork: true
+      containers:
+      - image: calico/typha:v0.2.2
+        name: calico-typha
+        ports:
+        - containerPort: 5473
+          name: calico-typha
+          protocol: TCP
+        env:
+          - name: TYPHA_LOGFILEPATH
+            value: "none"
+          - name: TYPHA_LOGSEVERITYSYS
+            value: "none"
+          - name: TYPHA_LOGSEVERITYSCREEN
+            value: "info"
+          - name: TYPHA_PROMETHEUSMETRICSENABLED
+            value: "true"
+          - name: TYPHA_CONNECTIONREBALANCINGMODE
+            value: "kubernetes"
+          - name: TYPHA_PROMETHEUSMETRICSPORT
+            value: "9093"
+          - name: TYPHA_DATASTORETYPE
+            value: "kubernetes"
+          - name: TYPHA_MAXCONNECTIONSLOWERLIMIT
+            value: "1"
+        volumeMounts:
+        - mountPath: /etc/calico
+          name: etc-calico
+          readOnly: true
+        resources:
+          requests:
+            cpu: __CALICO_TYPHA_CPU__
+      volumes:
+      - name: etc-calico
+        hostPath:
+          path: /etc/calico

cluster/addons/calico-policy-controller/typha-service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: calico-typha
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    k8s-app: calico-typha
+spec:
+  ports:
+    - port: 5473
+      protocol: TCP
+      targetPort: calico-typha
+      name: calico-typha
+  selector:
+    k8s-app: calico-typha

cluster/gce/container-linux/configure-helper.sh

@@ -34,6 +34,54 @@ function create-dirs {
   fi
 }
 
+# Vars assumed:
+#   NUM_NODES
+function get-calico-node-cpu {
+  local suggested_calico_cpus=100m
+  if [[ "${NUM_NODES}" -gt "10" ]]; then
+    suggested_calico_cpus=250m
+  fi
+  if [[ "${NUM_NODES}" -gt "100" ]]; then
+    suggested_calico_cpus=500m
+  fi
+  if [[ "${NUM_NODES}" -gt "500" ]]; then
+    suggested_calico_cpus=1000m
+  fi
+  echo "${suggested_calico_cpus}"
+}
+
+# Vars assumed:
+#    NUM_NODES
+function get-calico-typha-replicas {
+  local typha_count=1
+  if [[ "${NUM_NODES}" -gt "10" ]]; then
+    typha_count=2
+  fi
+  if [[ "${NUM_NODES}" -gt "100" ]]; then
+    typha_count=3
+  fi
+  if [[ "${NUM_NODES}" -gt "250" ]]; then
+    typha_count=4
+  fi
+  if [[ "${NUM_NODES}" -gt "500" ]]; then
+    typha_count=5
+  fi
+  echo "${typha_count}"
+}
+
+# Vars assumed:
+#    NUM_NODES
+function get-calico-typha-cpu {
+  local typha_cpu=200m
+  if [[ "${NUM_NODES}" -gt "10" ]]; then
+    typha_cpu=500m
+  fi
+  if [[ "${NUM_NODES}" -gt "100" ]]; then
+    typha_cpu=1000m
+  fi
+  echo "${typha_cpu}"
+}
+
 # Create directories referenced in the kube-controller-manager manifest for
 # bindmounts. This is used under the rkt runtime to work around
 # https://github.com/kubernetes/kubernetes/issues/26816
@@ -1213,9 +1261,13 @@ function start-kube-addons {
   if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
     setup-addon-manifests "addons" "calico-policy-controller"
 
-    # Replace the cluster cidr.
+    # Configure Calico based on cluster size and image type. 
-    local -r calico_file="${dst_dir}/calico-policy-controller/calico-node.yaml"
+    local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml"
-    sed -i -e "s@__CLUSTER_CIDR__@${CLUSTER_IP_RANGE}@g" "${calico_file}"
+    local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml"
+    sed -i -e "s@__CALICO_CNI_DIR__@/opt/cni/bin@g" "${ds_file}"
+    sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-node-cpu)@g" "${ds_file}"
+    sed -i -e "s@__CALICO_TYPHA_CPU__@$(get-calico-typha-cpu)@g" "${typha_dep_file}"
+    sed -i -e "s@__CALICO_TYPHA_REPLICAS__@$(get-calico-typha-replicas)@g" "${typha_dep_file}"
   fi
   if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then
     setup-addon-manifests "addons" "storage-class/gce"

cluster/gce/gci/configure-helper.sh

@@ -32,6 +32,55 @@ function setup-os-params {
   echo "core.%e.%p.%t" > /proc/sys/kernel/core_pattern
 }
 
+# Vars assumed:
+#   NUM_NODES
+function get-calico-node-cpu {
+  local suggested_calico_cpus=100m
+  if [[ "${NUM_NODES}" -gt "10" ]]; then
+    suggested_calico_cpus=250m
+  fi
+  if [[ "${NUM_NODES}" -gt "100" ]]; then
+    suggested_calico_cpus=500m
+  fi
+  if [[ "${NUM_NODES}" -gt "500" ]]; then
+    suggested_calico_cpus=1000m
+  fi
+  echo "${suggested_calico_cpus}"
+}
+
+# Vars assumed:
+#    NUM_NODES
+function get-calico-typha-replicas {
+  local typha_count=1
+  if [[ "${NUM_NODES}" -gt "10" ]]; then
+    typha_count=2
+  fi
+  if [[ "${NUM_NODES}" -gt "100" ]]; then
+    typha_count=3
+  fi
+  if [[ "${NUM_NODES}" -gt "250" ]]; then
+    typha_count=4
+  fi
+  if [[ "${NUM_NODES}" -gt "500" ]]; then
+    typha_count=5
+  fi
+  echo "${typha_count}"
+}
+
+# Vars assumed:
+#    NUM_NODES
+function get-calico-typha-cpu {
+  local typha_cpu=200m
+  if [[ "${NUM_NODES}" -gt "10" ]]; then
+    typha_cpu=500m
+  fi
+  if [[ "${NUM_NODES}" -gt "100" ]]; then
+    typha_cpu=1000m
+  fi
+  echo "${typha_cpu}"
+}
+
+
 function config-ip-firewall {
   echo "Configuring IP firewall rules"
   # The GCI image has host firewall which drop most inbound/forwarded packets.
@@ -1611,9 +1660,13 @@ function start-kube-addons {
   if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
     setup-addon-manifests "addons" "calico-policy-controller"
 
-    # Replace the cluster cidr.
+    # Configure Calico based on cluster size and image type. 
-    local -r calico_file="${dst_dir}/calico-policy-controller/calico-node.yaml"
+    local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml"
-    sed -i -e "s@__CLUSTER_CIDR__@${CLUSTER_IP_RANGE}@g" "${calico_file}"
+    local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml"
+    sed -i -e "s@__CALICO_CNI_DIR__@/home/kubernetes/bin@g" "${ds_file}"
+    sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-node-cpu)@g" "${ds_file}"
+    sed -i -e "s@__CALICO_TYPHA_CPU__@$(get-calico-typha-cpu)@g" "${typha_dep_file}"
+    sed -i -e "s@__CALICO_TYPHA_REPLICAS__@$(get-calico-typha-replicas)@g" "${typha_dep_file}"
   fi
   if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then
     setup-addon-manifests "addons" "storage-class/gce"