diff --git a/cluster/addons/calico-policy-controller/README.md b/cluster/addons/calico-policy-controller/README.md index 279f9fa7a9e..920cb84d2cf 100644 --- a/cluster/addons/calico-policy-controller/README.md +++ b/cluster/addons/calico-policy-controller/README.md @@ -6,7 +6,6 @@ Calico is an implementation of the Kubernetes network policy API. The provided - A DaemonSet which runs Calico on each node in the cluster. - A Deployment which installs the Calico Typha agent. - A Service for the Calico Typha agent. -- A HorizontalPodAutoscaler to dynamically adjust the Typha Deployment. ### Learn More diff --git a/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml b/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml index 8ec591bb598..48e1dac4498 100644 --- a/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml +++ b/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml @@ -28,7 +28,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: calico/node:v1.3.0-rc1 + image: calico/node:v1.3.0 env: - name: CALICO_DISABLE_FILE_LOGGING value: "true" @@ -128,7 +128,7 @@ spec: # Used to install CNI binaries. - name: cni-bin-dir hostPath: - path: /home/kubernetes/bin + path: __CALICO_CNI_DIR__ # Used to install CNI network config. - name: cni-net-dir hostPath: diff --git a/cluster/addons/calico-policy-controller/typha-autoscaler.yaml b/cluster/addons/calico-policy-controller/typha-autoscaler.yaml deleted file mode 100644 index 3b41c3ac0b4..00000000000 --- a/cluster/addons/calico-policy-controller/typha-autoscaler.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: autoscaling/v1 -kind: HorizontalPodAutoscaler -metadata: - name: calico-typha - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile - k8s-app: calico-typha -spec: - scaleTargetRef: - apiVersion: extensions/v1beta1 - kind: Deployment - name: calico-typha - minReplicas: 1 - maxReplicas: 20 - targetCPUUtilizationPercentage: 75 diff --git a/cluster/addons/calico-policy-controller/typha-deployment.yaml b/cluster/addons/calico-policy-controller/typha-deployment.yaml index 6c804e04753..8b4cf0ddc94 100644 --- a/cluster/addons/calico-policy-controller/typha-deployment.yaml +++ b/cluster/addons/calico-policy-controller/typha-deployment.yaml @@ -1,5 +1,5 @@ -kind: Deployment apiVersion: extensions/v1beta1 +kind: Deployment metadata: name: calico-typha namespace: kube-system @@ -8,7 +8,7 @@ metadata: addonmanager.kubernetes.io/mode: Reconcile k8s-app: calico-typha spec: - replicas: 1 + replicas: __CALICO_TYPHA_REPLICAS__ revisionHistoryLimit: 2 template: metadata: @@ -24,7 +24,7 @@ spec: projectcalico.org/ds-ready: "true" hostNetwork: true containers: - - image: calico/typha:v0.1.7 + - image: calico/typha:v0.2.2 name: calico-typha ports: - containerPort: 5473 @@ -39,10 +39,22 @@ spec: value: "info" - name: TYPHA_PROMETHEUSMETRICSENABLED value: "true" + - name: TYPHA_CONNECTIONREBALANCINGMODE + value: "kubernetes" - name: TYPHA_PROMETHEUSMETRICSPORT value: "9093" - name: TYPHA_DATASTORETYPE value: "kubernetes" + - name: TYPHA_MAXCONNECTIONSLOWERLIMIT + value: "1" + volumeMounts: + - mountPath: /etc/calico + name: etc-calico + readOnly: true resources: requests: - cpu: 200m + cpu: __CALICO_TYPHA_CPU__ + volumes: + - name: etc-calico + hostPath: + path: /etc/calico diff --git a/cluster/gce/container-linux/configure-helper.sh b/cluster/gce/container-linux/configure-helper.sh index ec70614dddb..78ec9dcdcdd 100755 --- a/cluster/gce/container-linux/configure-helper.sh +++ b/cluster/gce/container-linux/configure-helper.sh @@ -36,7 +36,7 @@ function create-dirs { # Vars assumed: # NUM_NODES -function get-calico-cpu { +function get-calico-node-cpu { local suggested_calico_cpus=100m if [[ "${NUM_NODES}" -gt "10" ]]; then suggested_calico_cpus=250m @@ -50,6 +50,38 @@ function get-calico-cpu { echo "${suggested_calico_cpus}" } +# Vars assumed: +# NUM_NODES +function get-calico-typha-replicas { + local typha_count=1 + if [[ "${NUM_NODES}" -gt "10" ]]; then + typha_count=2 + fi + if [[ "${NUM_NODES}" -gt "100" ]]; then + typha_count=3 + fi + if [[ "${NUM_NODES}" -gt "250" ]]; then + typha_count=4 + fi + if [[ "${NUM_NODES}" -gt "500" ]]; then + typha_count=5 + fi + echo "${typha_count}" +} + +# Vars assumed: +# NUM_NODES +function get-calico-typha-cpu { + local typha_cpu=200m + if [[ "${NUM_NODES}" -gt "10" ]]; then + typha_cpu=500m + fi + if [[ "${NUM_NODES}" -gt "100" ]]; then + typha_cpu=1000m + fi + echo "${typha_cpu}" +} + # Create directories referenced in the kube-controller-manager manifest for # bindmounts. This is used under the rkt runtime to work around # https://github.com/kubernetes/kubernetes/issues/26816 @@ -1229,9 +1261,13 @@ function start-kube-addons { if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then setup-addon-manifests "addons" "calico-policy-controller" - # Configure Calico resource requests based on cluster size. - local -r calico_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml" - sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-cpu)@g" "${calico_file}" + # Configure Calico based on cluster size and image type. + local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml" + local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml" + sed -i -e "s@__CALICO_CNI_DIR__@/opt/cni/bin@g" "${ds_file}" + sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-node-cpu)@g" "${ds_file}" + sed -i -e "s@__CALICO_TYPHA_CPU__@$(get-calico-typha-cpu)@g" "${typha_dep_file}" + sed -i -e "s@__CALICO_TYPHA_REPLICAS__@$(get-calico-typha-replicas)@g" "${typha_dep_file}" fi if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then setup-addon-manifests "addons" "storage-class/gce" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index c96ff6d782b..4e909533609 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -34,7 +34,7 @@ function setup-os-params { # Vars assumed: # NUM_NODES -function get-calico-cpu { +function get-calico-node-cpu { local suggested_calico_cpus=100m if [[ "${NUM_NODES}" -gt "10" ]]; then suggested_calico_cpus=250m @@ -48,6 +48,39 @@ function get-calico-cpu { echo "${suggested_calico_cpus}" } +# Vars assumed: +# NUM_NODES +function get-calico-typha-replicas { + local typha_count=1 + if [[ "${NUM_NODES}" -gt "10" ]]; then + typha_count=2 + fi + if [[ "${NUM_NODES}" -gt "100" ]]; then + typha_count=3 + fi + if [[ "${NUM_NODES}" -gt "250" ]]; then + typha_count=4 + fi + if [[ "${NUM_NODES}" -gt "500" ]]; then + typha_count=5 + fi + echo "${typha_count}" +} + +# Vars assumed: +# NUM_NODES +function get-calico-typha-cpu { + local typha_cpu=200m + if [[ "${NUM_NODES}" -gt "10" ]]; then + typha_cpu=500m + fi + if [[ "${NUM_NODES}" -gt "100" ]]; then + typha_cpu=1000m + fi + echo "${typha_cpu}" +} + + function config-ip-firewall { echo "Configuring IP firewall rules" # The GCI image has host firewall which drop most inbound/forwarded packets. @@ -1627,9 +1660,13 @@ function start-kube-addons { if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then setup-addon-manifests "addons" "calico-policy-controller" - # Configure Calico resource requests based on cluster size. - local -r calico_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml" - sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-cpu)@g" "${calico_file}" + # Configure Calico based on cluster size and image type. + local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml" + local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml" + sed -i -e "s@__CALICO_CNI_DIR__@/home/kubernetes/bin@g" "${ds_file}" + sed -i -e "s@__CALICO_NODE_CPU__@$(get-calico-node-cpu)@g" "${ds_file}" + sed -i -e "s@__CALICO_TYPHA_CPU__@$(get-calico-typha-cpu)@g" "${typha_dep_file}" + sed -i -e "s@__CALICO_TYPHA_REPLICAS__@$(get-calico-typha-replicas)@g" "${typha_dep_file}" fi if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then setup-addon-manifests "addons" "storage-class/gce"