From 8ef6b06d3982afda14de7335afa953e12ea8452d Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 31 May 2017 19:18:11 -0700 Subject: [PATCH 1/4] Use ip-masq-agent for MASQUERADE when using Calico policy --- cluster/addons/calico-policy-controller/README.md | 7 ------- cluster/addons/calico-policy-controller/calico-node.yaml | 6 ++---- cluster/gce/container-linux/configure-helper.sh | 4 ---- cluster/gce/gci/configure-helper.sh | 4 ---- 4 files changed, 2 insertions(+), 19 deletions(-) diff --git a/cluster/addons/calico-policy-controller/README.md b/cluster/addons/calico-policy-controller/README.md index e467ff1b7f9..ef8a94e3f34 100644 --- a/cluster/addons/calico-policy-controller/README.md +++ b/cluster/addons/calico-policy-controller/README.md @@ -3,13 +3,6 @@ Calico is an implementation of the Kubernetes network policy API. The provided manifest installs a DaemonSet which runs Calico on each node in the cluster. -### Templating - -The provided `calico-node.yaml` manifest includes the following placeholders which are populated -via templating. - -- `__CLUSTER_CIDR__`: The IP range from which Pod IP addresses are assigned. - ### Learn More Learn more about Calico at http://docs.projectcalico.org diff --git a/cluster/addons/calico-policy-controller/calico-node.yaml b/cluster/addons/calico-policy-controller/calico-node.yaml index 9244711d18c..ac4c4ba8c32 100644 --- a/cluster/addons/calico-policy-controller/calico-node.yaml +++ b/cluster/addons/calico-policy-controller/calico-node.yaml @@ -32,10 +32,6 @@ spec: value: "true" - name: CALICO_NETWORKING_BACKEND value: "none" - - name: CALICO_IPV4POOL_CIDR - value: "__CLUSTER_CIDR__" - - name: CALICO_IPV4POOL_IPIP - value: "off" - name: DATASTORE_TYPE value: "kubernetes" - name: FELIX_DEFAULTENDPOINTTOHOSTACTION @@ -48,6 +44,8 @@ spec: value: "true" - name: IP value: "" + - name: NO_DEFAULT_POOLS + value: "true" - name: NODENAME valueFrom: fieldRef: diff --git a/cluster/gce/container-linux/configure-helper.sh b/cluster/gce/container-linux/configure-helper.sh index 3ce39b8db91..d602dcc19ab 100755 --- a/cluster/gce/container-linux/configure-helper.sh +++ b/cluster/gce/container-linux/configure-helper.sh @@ -1212,10 +1212,6 @@ function start-kube-addons { fi if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then setup-addon-manifests "addons" "calico-policy-controller" - - # Replace the cluster cidr. - local -r calico_file="${dst_dir}/calico-policy-controller/calico-node.yaml" - sed -i -e "s@__CLUSTER_CIDR__@${CLUSTER_IP_RANGE}@g" "${calico_file}" fi if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then setup-addon-manifests "addons" "storage-class/gce" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 71068f9ffe4..428ef14eb4c 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1610,10 +1610,6 @@ function start-kube-addons { fi if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then setup-addon-manifests "addons" "calico-policy-controller" - - # Replace the cluster cidr. - local -r calico_file="${dst_dir}/calico-policy-controller/calico-node.yaml" - sed -i -e "s@__CLUSTER_CIDR__@${CLUSTER_IP_RANGE}@g" "${calico_file}" fi if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then setup-addon-manifests "addons" "storage-class/gce" From 88d3245671d6e3d0b80f5d2bf152eae5c1a439b1 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Wed, 7 Jun 2017 16:55:11 -0700 Subject: [PATCH 2/4] Add the Calico Typha agent --- .../addons/calico-policy-controller/README.md | 6 ++- ...o-node.yaml => calico-node-daemonset.yaml} | 10 ++-- .../typha-deployment.yaml | 48 +++++++++++++++++++ .../typha-service.yaml | 18 +++++++ 4 files changed, 77 insertions(+), 5 deletions(-) rename cluster/addons/calico-policy-controller/{calico-node.yaml => calico-node-daemonset.yaml} (95%) create mode 100644 cluster/addons/calico-policy-controller/typha-deployment.yaml create mode 100644 cluster/addons/calico-policy-controller/typha-service.yaml diff --git a/cluster/addons/calico-policy-controller/README.md b/cluster/addons/calico-policy-controller/README.md index ef8a94e3f34..920cb84d2cf 100644 --- a/cluster/addons/calico-policy-controller/README.md +++ b/cluster/addons/calico-policy-controller/README.md @@ -1,7 +1,11 @@ # Calico Policy Controller ============== -Calico is an implementation of the Kubernetes network policy API. The provided manifest installs a DaemonSet which runs Calico on each node in the cluster. +Calico is an implementation of the Kubernetes network policy API. The provided manifests install: + +- A DaemonSet which runs Calico on each node in the cluster. +- A Deployment which installs the Calico Typha agent. +- A Service for the Calico Typha agent. ### Learn More diff --git a/cluster/addons/calico-policy-controller/calico-node.yaml b/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml similarity index 95% rename from cluster/addons/calico-policy-controller/calico-node.yaml rename to cluster/addons/calico-policy-controller/calico-node-daemonset.yaml index ac4c4ba8c32..1f93c8ee9c2 100644 --- a/cluster/addons/calico-policy-controller/calico-node.yaml +++ b/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml @@ -26,7 +26,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: calico/node:v1.2.1 + image: calico/node:v1.3.0-rc1 env: - name: CALICO_DISABLE_FILE_LOGGING value: "true" @@ -34,6 +34,8 @@ spec: value: "none" - name: DATASTORE_TYPE value: "kubernetes" + - name: FELIX_TYPHAK8SSERVICENAME + value: "calico-typha" - name: FELIX_DEFAULTENDPOINTTOHOSTACTION value: "ACCEPT" - name: FELIX_IPV6SUPPORT @@ -67,7 +69,7 @@ spec: # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni - image: calico/cni:v1.8.3-hostport + image: calico/cni:v1.9.1 command: ["/install-cni.sh"] env: - name: CNI_CONF_NAME @@ -130,5 +132,5 @@ spec: hostPath: path: /etc/cni/net.d tolerations: - - key: "CriticalAddonsOnly" - operator: "Exists" + - key: "CriticalAddonsOnly" + operator: "Exists" diff --git a/cluster/addons/calico-policy-controller/typha-deployment.yaml b/cluster/addons/calico-policy-controller/typha-deployment.yaml new file mode 100644 index 00000000000..fb16b29c1ae --- /dev/null +++ b/cluster/addons/calico-policy-controller/typha-deployment.yaml @@ -0,0 +1,48 @@ +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + name: calico-typha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: calico-typha +spec: + replicas: 1 + revisionHistoryLimit: 2 + template: + metadata: + labels: + k8s-app: calico-typha + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + nodeSelector: + projectcalico.org/ds-ready: "true" + hostNetwork: true + containers: + - image: calico/typha:v0.1.7 + name: calico-typha + ports: + - containerPort: 5473 + name: calico-typha + protocol: TCP + env: + - name: TYPHA_LOGFILEPATH + value: "none" + - name: TYPHA_LOGSEVERITYSYS + value: "none" + - name: TYPHA_LOGSEVERITYSCREEN + value: "info" + - name: TYPHA_PROMETHEUSMETRICSENABLED + value: "true" + - name: TYPHA_PROMETHEUSMETRICSPORT + value: "9093" + - name: TYPHA_DATASTORETYPE + value: "kubernetes" + resources: + requests: + cpu: 1000m diff --git a/cluster/addons/calico-policy-controller/typha-service.yaml b/cluster/addons/calico-policy-controller/typha-service.yaml new file mode 100644 index 00000000000..a1281db2158 --- /dev/null +++ b/cluster/addons/calico-policy-controller/typha-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: calico-typha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: calico-typha +spec: + ports: + - port: 5473 + protocol: TCP + targetPort: calico-typha + name: calico-typha + selector: + k8s-app: calico-typha + From 83ec0d87ffcbc67c700c25dc900f156f3beec856 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Thu, 8 Jun 2017 13:52:56 -0700 Subject: [PATCH 3/4] Make calico/node resource requests dynamic based on cluster size --- .../addons/calico-policy-controller/README.md | 1 + .../calico-node-daemonset.yaml | 4 +++- .../typha-autoscaler.yaml | 17 ++++++++++++++++ .../typha-deployment.yaml | 2 +- .../typha-service.yaml | 1 - .../gce/container-linux/configure-helper.sh | 20 +++++++++++++++++++ cluster/gce/gci/configure-helper.sh | 20 +++++++++++++++++++ 7 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 cluster/addons/calico-policy-controller/typha-autoscaler.yaml diff --git a/cluster/addons/calico-policy-controller/README.md b/cluster/addons/calico-policy-controller/README.md index 920cb84d2cf..279f9fa7a9e 100644 --- a/cluster/addons/calico-policy-controller/README.md +++ b/cluster/addons/calico-policy-controller/README.md @@ -6,6 +6,7 @@ Calico is an implementation of the Kubernetes network policy API. The provided - A DaemonSet which runs Calico on each node in the cluster. - A Deployment which installs the Calico Typha agent. - A Service for the Calico Typha agent. +- A HorizontalPodAutoscaler to dynamically adjust the Typha Deployment. ### Learn More diff --git a/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml b/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml index 1f93c8ee9c2..8ec591bb598 100644 --- a/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml +++ b/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml @@ -11,6 +11,8 @@ spec: selector: matchLabels: k8s-app: calico-node + updateStrategy: + type: RollingUpdate template: metadata: labels: @@ -58,7 +60,7 @@ spec: privileged: true resources: requests: - cpu: 250m + cpu: __CALICO_NODE_CPU__ volumeMounts: - mountPath: /lib/modules name: lib-modules diff --git a/cluster/addons/calico-policy-controller/typha-autoscaler.yaml b/cluster/addons/calico-policy-controller/typha-autoscaler.yaml new file mode 100644 index 00000000000..3b41c3ac0b4 --- /dev/null +++ b/cluster/addons/calico-policy-controller/typha-autoscaler.yaml @@ -0,0 +1,17 @@ +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + name: calico-typha + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: calico-typha +spec: + scaleTargetRef: + apiVersion: extensions/v1beta1 + kind: Deployment + name: calico-typha + minReplicas: 1 + maxReplicas: 20 + targetCPUUtilizationPercentage: 75 diff --git a/cluster/addons/calico-policy-controller/typha-deployment.yaml b/cluster/addons/calico-policy-controller/typha-deployment.yaml index fb16b29c1ae..6c804e04753 100644 --- a/cluster/addons/calico-policy-controller/typha-deployment.yaml +++ b/cluster/addons/calico-policy-controller/typha-deployment.yaml @@ -45,4 +45,4 @@ spec: value: "kubernetes" resources: requests: - cpu: 1000m + cpu: 200m diff --git a/cluster/addons/calico-policy-controller/typha-service.yaml b/cluster/addons/calico-policy-controller/typha-service.yaml index a1281db2158..c9c2b382afb 100644 --- a/cluster/addons/calico-policy-controller/typha-service.yaml +++ b/cluster/addons/calico-policy-controller/typha-service.yaml @@ -15,4 +15,3 @@ spec: name: calico-typha selector: k8s-app: calico-typha - diff --git a/cluster/gce/container-linux/configure-helper.sh b/cluster/gce/container-linux/configure-helper.sh index d602dcc19ab..ec70614dddb 100755 --- a/cluster/gce/container-linux/configure-helper.sh +++ b/cluster/gce/container-linux/configure-helper.sh @@ -34,6 +34,22 @@ function create-dirs { fi } +# Vars assumed: +# NUM_NODES +function get-calico-cpu { + local suggested_calico_cpus=100m + if [[ "${NUM_NODES}" -gt "10" ]]; then + suggested_calico_cpus=250m + fi + if [[ "${NUM_NODES}" -gt "100" ]]; then + suggested_calico_cpus=500m + fi + if [[ "${NUM_NODES}" -gt "500" ]]; then + suggested_calico_cpus=1000m + fi + echo "${suggested_calico_cpus}" +} + # Create directories referenced in the kube-controller-manager manifest for # bindmounts. This is used under the rkt runtime to work around # https://github.com/kubernetes/kubernetes/issues/26816 @@ -1212,6 +1228,10 @@ function start-kube-addons { fi if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then setup-addon-manifests "addons" "calico-policy-controller" + + # Configure Calico resource requests based on cluster size. + local -r calico_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml" + sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-cpu)@g" "${calico_file}" fi if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then setup-addon-manifests "addons" "storage-class/gce" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 428ef14eb4c..c96ff6d782b 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -32,6 +32,22 @@ function setup-os-params { echo "core.%e.%p.%t" > /proc/sys/kernel/core_pattern } +# Vars assumed: +# NUM_NODES +function get-calico-cpu { + local suggested_calico_cpus=100m + if [[ "${NUM_NODES}" -gt "10" ]]; then + suggested_calico_cpus=250m + fi + if [[ "${NUM_NODES}" -gt "100" ]]; then + suggested_calico_cpus=500m + fi + if [[ "${NUM_NODES}" -gt "500" ]]; then + suggested_calico_cpus=1000m + fi + echo "${suggested_calico_cpus}" +} + function config-ip-firewall { echo "Configuring IP firewall rules" # The GCI image has host firewall which drop most inbound/forwarded packets. @@ -1610,6 +1626,10 @@ function start-kube-addons { fi if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then setup-addon-manifests "addons" "calico-policy-controller" + + # Configure Calico resource requests based on cluster size. + local -r calico_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml" + sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-cpu)@g" "${calico_file}" fi if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then setup-addon-manifests "addons" "storage-class/gce" From 948c6c8027f47f0ac28098976dcd464770f785d4 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Sat, 10 Jun 2017 16:58:16 -0700 Subject: [PATCH 4/4] Change how Typha CPU / replias are determined. --- .../addons/calico-policy-controller/README.md | 1 - .../calico-node-daemonset.yaml | 4 +- .../typha-autoscaler.yaml | 17 ------- .../typha-deployment.yaml | 20 +++++++-- .../gce/container-linux/configure-helper.sh | 44 ++++++++++++++++-- cluster/gce/gci/configure-helper.sh | 45 +++++++++++++++++-- 6 files changed, 99 insertions(+), 32 deletions(-) delete mode 100644 cluster/addons/calico-policy-controller/typha-autoscaler.yaml diff --git a/cluster/addons/calico-policy-controller/README.md b/cluster/addons/calico-policy-controller/README.md index 279f9fa7a9e..920cb84d2cf 100644 --- a/cluster/addons/calico-policy-controller/README.md +++ b/cluster/addons/calico-policy-controller/README.md @@ -6,7 +6,6 @@ Calico is an implementation of the Kubernetes network policy API. The provided - A DaemonSet which runs Calico on each node in the cluster. - A Deployment which installs the Calico Typha agent. - A Service for the Calico Typha agent. -- A HorizontalPodAutoscaler to dynamically adjust the Typha Deployment. ### Learn More diff --git a/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml b/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml index 8ec591bb598..48e1dac4498 100644 --- a/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml +++ b/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml @@ -28,7 +28,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: calico/node:v1.3.0-rc1 + image: calico/node:v1.3.0 env: - name: CALICO_DISABLE_FILE_LOGGING value: "true" @@ -128,7 +128,7 @@ spec: # Used to install CNI binaries. - name: cni-bin-dir hostPath: - path: /home/kubernetes/bin + path: __CALICO_CNI_DIR__ # Used to install CNI network config. - name: cni-net-dir hostPath: diff --git a/cluster/addons/calico-policy-controller/typha-autoscaler.yaml b/cluster/addons/calico-policy-controller/typha-autoscaler.yaml deleted file mode 100644 index 3b41c3ac0b4..00000000000 --- a/cluster/addons/calico-policy-controller/typha-autoscaler.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: autoscaling/v1 -kind: HorizontalPodAutoscaler -metadata: - name: calico-typha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile - k8s-app: calico-typha -spec: - scaleTargetRef: - apiVersion: extensions/v1beta1 - kind: Deployment - name: calico-typha - minReplicas: 1 - maxReplicas: 20 - targetCPUUtilizationPercentage: 75 diff --git a/cluster/addons/calico-policy-controller/typha-deployment.yaml b/cluster/addons/calico-policy-controller/typha-deployment.yaml index 6c804e04753..8b4cf0ddc94 100644 --- a/cluster/addons/calico-policy-controller/typha-deployment.yaml +++ b/cluster/addons/calico-policy-controller/typha-deployment.yaml @@ -1,5 +1,5 @@ -kind: Deployment apiVersion: extensions/v1beta1 +kind: Deployment metadata: name: calico-typha namespace: kube-system @@ -8,7 +8,7 @@ metadata: addonmanager.kubernetes.io/mode: Reconcile k8s-app: calico-typha spec: - replicas: 1 + replicas: __CALICO_TYPHA_REPLICAS__ revisionHistoryLimit: 2 template: metadata: @@ -24,7 +24,7 @@ spec: projectcalico.org/ds-ready: "true" hostNetwork: true containers: - - image: calico/typha:v0.1.7 + - image: calico/typha:v0.2.2 name: calico-typha ports: - containerPort: 5473 @@ -39,10 +39,22 @@ spec: value: "info" - name: TYPHA_PROMETHEUSMETRICSENABLED value: "true" + - name: TYPHA_CONNECTIONREBALANCINGMODE + value: "kubernetes" - name: TYPHA_PROMETHEUSMETRICSPORT value: "9093" - name: TYPHA_DATASTORETYPE value: "kubernetes" + - name: TYPHA_MAXCONNECTIONSLOWERLIMIT + value: "1" + volumeMounts: + - mountPath: /etc/calico + name: etc-calico + readOnly: true resources: requests: - cpu: 200m + cpu: __CALICO_TYPHA_CPU__ + volumes: + - name: etc-calico + hostPath: + path: /etc/calico diff --git a/cluster/gce/container-linux/configure-helper.sh b/cluster/gce/container-linux/configure-helper.sh index ec70614dddb..78ec9dcdcdd 100755 --- a/cluster/gce/container-linux/configure-helper.sh +++ b/cluster/gce/container-linux/configure-helper.sh @@ -36,7 +36,7 @@ function create-dirs { # Vars assumed: # NUM_NODES -function get-calico-cpu { +function get-calico-node-cpu { local suggested_calico_cpus=100m if [[ "${NUM_NODES}" -gt "10" ]]; then suggested_calico_cpus=250m @@ -50,6 +50,38 @@ function get-calico-cpu { echo "${suggested_calico_cpus}" } +# Vars assumed: +# NUM_NODES +function get-calico-typha-replicas { + local typha_count=1 + if [[ "${NUM_NODES}" -gt "10" ]]; then + typha_count=2 + fi + if [[ "${NUM_NODES}" -gt "100" ]]; then + typha_count=3 + fi + if [[ "${NUM_NODES}" -gt "250" ]]; then + typha_count=4 + fi + if [[ "${NUM_NODES}" -gt "500" ]]; then + typha_count=5 + fi + echo "${typha_count}" +} + +# Vars assumed: +# NUM_NODES +function get-calico-typha-cpu { + local typha_cpu=200m + if [[ "${NUM_NODES}" -gt "10" ]]; then + typha_cpu=500m + fi + if [[ "${NUM_NODES}" -gt "100" ]]; then + typha_cpu=1000m + fi + echo "${typha_cpu}" +} + # Create directories referenced in the kube-controller-manager manifest for # bindmounts. This is used under the rkt runtime to work around # https://github.com/kubernetes/kubernetes/issues/26816 @@ -1229,9 +1261,13 @@ function start-kube-addons { if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then setup-addon-manifests "addons" "calico-policy-controller" - # Configure Calico resource requests based on cluster size. - local -r calico_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml" - sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-cpu)@g" "${calico_file}" + # Configure Calico based on cluster size and image type. + local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml" + local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml" + sed -i -e "s@__CALICO_CNI_DIR__@/opt/cni/bin@g" "${ds_file}" + sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-node-cpu)@g" "${ds_file}" + sed -i -e "s@__CALICO_TYPHA_CPU__@$(get-calico-typha-cpu)@g" "${typha_dep_file}" + sed -i -e "s@__CALICO_TYPHA_REPLICAS__@$(get-calico-typha-replicas)@g" "${typha_dep_file}" fi if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then setup-addon-manifests "addons" "storage-class/gce" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index c96ff6d782b..4e909533609 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -34,7 +34,7 @@ function setup-os-params { # Vars assumed: # NUM_NODES -function get-calico-cpu { +function get-calico-node-cpu { local suggested_calico_cpus=100m if [[ "${NUM_NODES}" -gt "10" ]]; then suggested_calico_cpus=250m @@ -48,6 +48,39 @@ function get-calico-cpu { echo "${suggested_calico_cpus}" } +# Vars assumed: +# NUM_NODES +function get-calico-typha-replicas { + local typha_count=1 + if [[ "${NUM_NODES}" -gt "10" ]]; then + typha_count=2 + fi + if [[ "${NUM_NODES}" -gt "100" ]]; then + typha_count=3 + fi + if [[ "${NUM_NODES}" -gt "250" ]]; then + typha_count=4 + fi + if [[ "${NUM_NODES}" -gt "500" ]]; then + typha_count=5 + fi + echo "${typha_count}" +} + +# Vars assumed: +# NUM_NODES +function get-calico-typha-cpu { + local typha_cpu=200m + if [[ "${NUM_NODES}" -gt "10" ]]; then + typha_cpu=500m + fi + if [[ "${NUM_NODES}" -gt "100" ]]; then + typha_cpu=1000m + fi + echo "${typha_cpu}" +} + + function config-ip-firewall { echo "Configuring IP firewall rules" # The GCI image has host firewall which drop most inbound/forwarded packets. @@ -1627,9 +1660,13 @@ function start-kube-addons { if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then setup-addon-manifests "addons" "calico-policy-controller" - # Configure Calico resource requests based on cluster size. - local -r calico_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml" - sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-cpu)@g" "${calico_file}" + # Configure Calico based on cluster size and image type. + local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml" + local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml" + sed -i -e "s@__CALICO_CNI_DIR__@/home/kubernetes/bin@g" "${ds_file}" + sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-node-cpu)@g" "${ds_file}" + sed -i -e "s@__CALICO_TYPHA_CPU__@$(get-calico-typha-cpu)@g" "${typha_dep_file}" + sed -i -e "s@__CALICO_TYPHA_REPLICAS__@$(get-calico-typha-replicas)@g" "${typha_dep_file}" fi if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then setup-addon-manifests "addons" "storage-class/gce"