Change how Typha CPU / replias are determined.

cluster/addons/calico-policy-controller/README.md

@@ -6,7 +6,6 @@ Calico is an implementation of the Kubernetes network policy API.  The provided
 - A DaemonSet which runs Calico on each node in the cluster.
 - A Deployment which installs the Calico Typha agent.
 - A Service for the Calico Typha agent.
-- A HorizontalPodAutoscaler to dynamically adjust the Typha Deployment.
 
 ### Learn More
 

cluster/addons/calico-policy-controller/calico-node-daemonset.yaml

@@ -28,7 +28,7 @@ spec:
         # container programs network policy and routes on each
         # host.
         - name: calico-node
-          image: calico/node:v1.3.0-rc1
+          image: calico/node:v1.3.0
           env:
             - name: CALICO_DISABLE_FILE_LOGGING
               value: "true"
@@ -128,7 +128,7 @@ spec:
         # Used to install CNI binaries.
         - name: cni-bin-dir
           hostPath:
-            path: /home/kubernetes/bin
+            path: __CALICO_CNI_DIR__
         # Used to install CNI network config.
         - name: cni-net-dir
           hostPath:

cluster/addons/calico-policy-controller/typha-autoscaler.yaml

@@ -1,17 +0,0 @@
-apiVersion: autoscaling/v1
-kind: HorizontalPodAutoscaler
-metadata:
-  name: calico-typha
-  namespace: kube-system
-  labels:
-    kubernetes.io/cluster-service: "true"
-    addonmanager.kubernetes.io/mode: Reconcile
-    k8s-app: calico-typha
-spec:
-  scaleTargetRef:
-    apiVersion: extensions/v1beta1
-    kind: Deployment
-    name: calico-typha
-  minReplicas: 1
-  maxReplicas: 20
-  targetCPUUtilizationPercentage: 75

cluster/addons/calico-policy-controller/typha-deployment.yaml

@@ -1,5 +1,5 @@
-kind: Deployment
 apiVersion: extensions/v1beta1
+kind: Deployment
 metadata:
   name: calico-typha
   namespace: kube-system
@@ -8,7 +8,7 @@ metadata:
     addonmanager.kubernetes.io/mode: Reconcile
     k8s-app: calico-typha
 spec:
-  replicas: 1
+  replicas: __CALICO_TYPHA_REPLICAS__
   revisionHistoryLimit: 2
   template:
     metadata:
@@ -24,7 +24,7 @@ spec:
         projectcalico.org/ds-ready: "true"
       hostNetwork: true
       containers:
-      - image: calico/typha:v0.1.7
+      - image: calico/typha:v0.2.2
         name: calico-typha
         ports:
         - containerPort: 5473
@@ -39,10 +39,22 @@ spec:
             value: "info"
           - name: TYPHA_PROMETHEUSMETRICSENABLED
             value: "true"
+          - name: TYPHA_CONNECTIONREBALANCINGMODE
+            value: "kubernetes"
           - name: TYPHA_PROMETHEUSMETRICSPORT
             value: "9093"
           - name: TYPHA_DATASTORETYPE
             value: "kubernetes"
+          - name: TYPHA_MAXCONNECTIONSLOWERLIMIT
+            value: "1"
+        volumeMounts:
+        - mountPath: /etc/calico
+          name: etc-calico
+          readOnly: true
         resources:
           requests:
-            cpu: 200m
+            cpu: __CALICO_TYPHA_CPU__
+      volumes:
+      - name: etc-calico
+        hostPath:
+          path: /etc/calico

cluster/gce/container-linux/configure-helper.sh

@@ -36,7 +36,7 @@ function create-dirs {
 
 # Vars assumed:
 #   NUM_NODES
-function get-calico-cpu {
+function get-calico-node-cpu {
   local suggested_calico_cpus=100m
   if [[ "${NUM_NODES}" -gt "10" ]]; then
     suggested_calico_cpus=250m
@@ -50,6 +50,38 @@ function get-calico-cpu {
   echo "${suggested_calico_cpus}"
 }
 
+# Vars assumed:
+#    NUM_NODES
+function get-calico-typha-replicas {
+  local typha_count=1
+  if [[ "${NUM_NODES}" -gt "10" ]]; then
+    typha_count=2
+  fi
+  if [[ "${NUM_NODES}" -gt "100" ]]; then
+    typha_count=3
+  fi
+  if [[ "${NUM_NODES}" -gt "250" ]]; then
+    typha_count=4
+  fi
+  if [[ "${NUM_NODES}" -gt "500" ]]; then
+    typha_count=5
+  fi
+  echo "${typha_count}"
+}
+
+# Vars assumed:
+#    NUM_NODES
+function get-calico-typha-cpu {
+  local typha_cpu=200m
+  if [[ "${NUM_NODES}" -gt "10" ]]; then
+    typha_cpu=500m
+  fi
+  if [[ "${NUM_NODES}" -gt "100" ]]; then
+    typha_cpu=1000m
+  fi
+  echo "${typha_cpu}"
+}
+
 # Create directories referenced in the kube-controller-manager manifest for
 # bindmounts. This is used under the rkt runtime to work around
 # https://github.com/kubernetes/kubernetes/issues/26816
@@ -1229,9 +1261,13 @@ function start-kube-addons {
   if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
     setup-addon-manifests "addons" "calico-policy-controller"
 
-    # Configure Calico resource requests based on cluster size.
+    # Configure Calico based on cluster size and image type. 
-    local -r calico_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml"
+    local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml"
-    sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-cpu)@g" "${calico_file}"
+    local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml"
+    sed -i -e "s@__CALICO_CNI_DIR__@/opt/cni/bin@g" "${ds_file}"
+    sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-node-cpu)@g" "${ds_file}"
+    sed -i -e "s@__CALICO_TYPHA_CPU__@$(get-calico-typha-cpu)@g" "${typha_dep_file}"
+    sed -i -e "s@__CALICO_TYPHA_REPLICAS__@$(get-calico-typha-replicas)@g" "${typha_dep_file}"
   fi
   if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then
     setup-addon-manifests "addons" "storage-class/gce"

cluster/gce/gci/configure-helper.sh

@@ -34,7 +34,7 @@ function setup-os-params {
 
 # Vars assumed:
 #   NUM_NODES
-function get-calico-cpu {
+function get-calico-node-cpu {
   local suggested_calico_cpus=100m
   if [[ "${NUM_NODES}" -gt "10" ]]; then
     suggested_calico_cpus=250m
@@ -48,6 +48,39 @@ function get-calico-cpu {
   echo "${suggested_calico_cpus}"
 }
 
+# Vars assumed:
+#    NUM_NODES
+function get-calico-typha-replicas {
+  local typha_count=1
+  if [[ "${NUM_NODES}" -gt "10" ]]; then
+    typha_count=2
+  fi
+  if [[ "${NUM_NODES}" -gt "100" ]]; then
+    typha_count=3
+  fi
+  if [[ "${NUM_NODES}" -gt "250" ]]; then
+    typha_count=4
+  fi
+  if [[ "${NUM_NODES}" -gt "500" ]]; then
+    typha_count=5
+  fi
+  echo "${typha_count}"
+}
+
+# Vars assumed:
+#    NUM_NODES
+function get-calico-typha-cpu {
+  local typha_cpu=200m
+  if [[ "${NUM_NODES}" -gt "10" ]]; then
+    typha_cpu=500m
+  fi
+  if [[ "${NUM_NODES}" -gt "100" ]]; then
+    typha_cpu=1000m
+  fi
+  echo "${typha_cpu}"
+}
+
+
 function config-ip-firewall {
   echo "Configuring IP firewall rules"
   # The GCI image has host firewall which drop most inbound/forwarded packets.
@@ -1627,9 +1660,13 @@ function start-kube-addons {
   if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
     setup-addon-manifests "addons" "calico-policy-controller"
 
-    # Configure Calico resource requests based on cluster size.
+    # Configure Calico based on cluster size and image type. 
-    local -r calico_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml"
+    local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml"
-    sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-cpu)@g" "${calico_file}"
+    local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml"
+    sed -i -e "s@__CALICO_CNI_DIR__@/home/kubernetes/bin@g" "${ds_file}"
+    sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-node-cpu)@g" "${ds_file}"
+    sed -i -e "s@__CALICO_TYPHA_CPU__@$(get-calico-typha-cpu)@g" "${typha_dep_file}"
+    sed -i -e "s@__CALICO_TYPHA_REPLICAS__@$(get-calico-typha-replicas)@g" "${typha_dep_file}"
   fi
   if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then
     setup-addon-manifests "addons" "storage-class/gce"