From 63744a819f1a3daaf567464b8fa03ef2763f18f9 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Sun, 14 Aug 2016 11:43:46 -0700 Subject: [PATCH] Update Calico add-on --- .../addons/calico-policy-controller/README.md | 14 +- .../calico-etcd-service.yaml | 15 -- .../calico-etcd-statefulset.yaml | 42 ------ .../calico-policy-controller/calico-node.yaml | 136 ++++++++++++++++++ .../calico-policy-controller.yaml | 32 ----- cluster/gce/config-default.sh | 6 + cluster/gce/config-test.sh | 6 + .../gce/container-linux/configure-helper.sh | 4 + cluster/gce/gci/configure-helper.sh | 16 ++- cluster/saltbase/salt/calico/10-calico.conf | 16 --- .../saltbase/salt/calico/calico-node.manifest | 40 ------ cluster/saltbase/salt/calico/master.sls | 19 --- cluster/saltbase/salt/calico/node.sls | 32 ----- cluster/saltbase/salt/kubelet/default | 2 +- cluster/saltbase/salt/top.sls | 3 - 15 files changed, 176 insertions(+), 207 deletions(-) delete mode 100644 cluster/addons/calico-policy-controller/calico-etcd-service.yaml delete mode 100644 cluster/addons/calico-policy-controller/calico-etcd-statefulset.yaml create mode 100644 cluster/addons/calico-policy-controller/calico-node.yaml delete mode 100644 cluster/addons/calico-policy-controller/calico-policy-controller.yaml delete mode 100644 cluster/saltbase/salt/calico/10-calico.conf delete mode 100644 cluster/saltbase/salt/calico/calico-node.manifest delete mode 100644 cluster/saltbase/salt/calico/master.sls diff --git a/cluster/addons/calico-policy-controller/README.md b/cluster/addons/calico-policy-controller/README.md index 473899338ab..e467ff1b7f9 100644 --- a/cluster/addons/calico-policy-controller/README.md +++ b/cluster/addons/calico-policy-controller/README.md @@ -1,11 +1,17 @@ # Calico Policy Controller ============== -Calico Policy Controller is an implementation of the Kubernetes network policy API. +Calico is an implementation of the Kubernetes network policy API. The provided manifest installs a DaemonSet which runs Calico on each node in the cluster. -Learn more at: -- https://github.com/projectcalico/k8s-policy -- http://kubernetes.io/docs/user-guide/networkpolicies/ +### Templating +The provided `calico-node.yaml` manifest includes the following placeholders which are populated +via templating. + +- `__CLUSTER_CIDR__`: The IP range from which Pod IP addresses are assigned. + +### Learn More + +Learn more about Calico at http://docs.projectcalico.org [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/calico-policy-controller/README.md?pixel)]() diff --git a/cluster/addons/calico-policy-controller/calico-etcd-service.yaml b/cluster/addons/calico-policy-controller/calico-etcd-service.yaml deleted file mode 100644 index ad9a95e4690..00000000000 --- a/cluster/addons/calico-policy-controller/calico-etcd-service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - k8s-app: calico-etcd - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile - name: calico-etcd - namespace: kube-system -spec: - clusterIP: 10.0.0.17 - ports: - - port: 6666 - selector: - k8s-app: calico-etcd diff --git a/cluster/addons/calico-policy-controller/calico-etcd-statefulset.yaml b/cluster/addons/calico-policy-controller/calico-etcd-statefulset.yaml deleted file mode 100644 index 61e996ce195..00000000000 --- a/cluster/addons/calico-policy-controller/calico-etcd-statefulset.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: "apps/v1beta1" -kind: StatefulSet -metadata: - name: calico-etcd - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile - k8s-app: calico-etcd -spec: - serviceName: calico-etcd - replicas: 1 - template: - metadata: - labels: - kubernetes.io/cluster-service: "true" - k8s-app: calico-etcd - spec: - hostNetwork: true - containers: - - name: calico-etcd - image: gcr.io/google_containers/etcd:2.2.1 - env: - - name: CALICO_ETCD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - command: ["/bin/sh","-c"] - args: ["/usr/local/bin/etcd --name=calico --data-dir=/var/etcd/calico-data --advertise-client-urls=http://$CALICO_ETCD_IP:6666 --listen-client-urls=http://0.0.0.0:6666 --listen-peer-urls=http://0.0.0.0:6667"] - volumeMounts: - - name: var-etcd - mountPath: /var/etcd - volumeClaimTemplates: - - metadata: - name: var-etcd - annotations: - volume.alpha.kubernetes.io/storage-class: anything - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi diff --git a/cluster/addons/calico-policy-controller/calico-node.yaml b/cluster/addons/calico-policy-controller/calico-node.yaml new file mode 100644 index 00000000000..94ce5ea4e15 --- /dev/null +++ b/cluster/addons/calico-policy-controller/calico-node.yaml @@ -0,0 +1,136 @@ +kind: DaemonSet +apiVersion: extensions/v1beta1 +metadata: + name: calico-node + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: calico-node +spec: + selector: + matchLabels: + k8s-app: calico-node + template: + metadata: + labels: + k8s-app: calico-node + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: | + [{"key": "dedicated", "value": "master", "effect": "NoSchedule" }, + {"key":"CriticalAddonsOnly", "operator":"Exists"}] + spec: + nodeSelector: + projectcalico.org/ds-ready: "true" + hostNetwork: true + containers: + # Runs calico/node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: calico/node:v1.2.1 + env: + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + - name: CALICO_NETWORKING_BACKEND + value: "none" + - name: CALICO_IPV4POOL_CIDR + value: "__CLUSTER_CIDR__" + - name: CALICO_IPV4POOL_IPIP + value: "off" + - name: DATASTORE_TYPE + value: "kubernetes" + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + - name: FELIX_IPV6SUPPORT + value: "false" + - name: FELIX_LOGSEVERITYSYS + value: "none" + - name: FELIX_PROMETHEUSMETRICSENABLED + value: "true" + - name: IP + value: "" + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: WAIT_FOR_DATASTORE + value: "true" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /etc/calico + name: etc-calico + readOnly: true + # This container installs the Calico CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: calico/cni:v1.8.3-hostport + command: ["/install-cni.sh"] + env: + - name: CNI_CONF_NAME + value: "10-calico.conflist" + - name: CNI_NETWORK_CONFIG + value: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.3.0", + "plugins": [ + { + "type": "calico", + "log_level": "debug", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "ipam": { + "type": "host-local", + "subnet": "usePodCidr" + }, + "policy": { + "type": "k8s", + "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__" + }, + "kubernetes": { + "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__", + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + }, + { + "type": "portmap", + "capabilities": {"portMappings": true}, + "noSnat": true + } + ] + } + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + volumes: + # Used to ensure proper kmods are installed. + - name: lib-modules + hostPath: + path: /lib/modules + # Mount in the Felix config file from the host. + - name: etc-calico + hostPath: + path: /etc/calico + # Used to install CNI binaries. + - name: cni-bin-dir + hostPath: + path: /home/kubernetes/bin + # Used to install CNI network config. + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d diff --git a/cluster/addons/calico-policy-controller/calico-policy-controller.yaml b/cluster/addons/calico-policy-controller/calico-policy-controller.yaml deleted file mode 100644 index 899cadeabf1..00000000000 --- a/cluster/addons/calico-policy-controller/calico-policy-controller.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: calico-policy-controller - namespace: kube-system - labels: - k8s-app: calico-policy - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile -spec: - replicas: 1 - selector: - k8s-app: calico-policy - template: - metadata: - name: calico-policy-controller - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - k8s-app: calico-policy - spec: - hostNetwork: true - containers: - - name: calico-policy-controller - image: calico/kube-policy-controller:v0.2.0 - env: - - name: ETCD_ENDPOINTS - value: "http://10.0.0.17:6666" - - name: K8S_API - value: "https://kubernetes.default:443" - - name: CONFIGURE_ETC_HOSTS - value: "true" diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 00af1812954..e707b5e5364 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -124,6 +124,12 @@ ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-influxdb}" # TODO(piosz): remove this in 1.8 NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}" +# To avoid running Calico on a node that is not configured appropriately, +# label each Node so that the DaemonSet can run the Pods only on ready Nodes. +if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then + NODE_LABELS="$NODE_LABELS,projectcalico.org/ds-ready=true" +fi + # Optional: Enable node logging. ENABLE_NODE_LOGGING="${KUBE_ENABLE_NODE_LOGGING:-true}" LOGGING_DESTINATION="${KUBE_LOGGING_DESTINATION:-gcp}" # options: elasticsearch, gcp diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 4c82a048c6a..dadeb69fe40 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -161,6 +161,12 @@ KUBEPROXY_TEST_ARGS="${KUBEPROXY_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}" # TODO(piosz): remove this in 1.8 NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}" +# To avoid running Calico on a node that is not configured appropriately, +# label each Node so that the DaemonSet can run the Pods only on ready Nodes. +if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then + NODE_LABELS="$NODE_LABELS,projectcalico.org/ds-ready=true" +fi + # Optional: Enable node logging. ENABLE_NODE_LOGGING="${KUBE_ENABLE_NODE_LOGGING:-true}" LOGGING_DESTINATION="${KUBE_LOGGING_DESTINATION:-gcp}" # options: elasticsearch, gcp diff --git a/cluster/gce/container-linux/configure-helper.sh b/cluster/gce/container-linux/configure-helper.sh index 4147270370f..fca37a92db8 100755 --- a/cluster/gce/container-linux/configure-helper.sh +++ b/cluster/gce/container-linux/configure-helper.sh @@ -1201,6 +1201,10 @@ function start-kube-addons { fi if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then setup-addon-manifests "addons" "calico-policy-controller" + + # Replace the cluster cidr. + local -r calico_file="${dst_dir}/calico-policy-controller/calico-node.yaml" + sed -i -e "s@__CLUSTER_CIDR__@${CLUSTER_IP_RANGE}@g" "${calico_file}" fi if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then setup-addon-manifests "addons" "storage-class/gce" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index f6c63049730..1f1a0a1e6a0 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -701,13 +701,19 @@ function start-kubelet { flags+=" --anonymous-auth=false --authorization-mode=Webhook --client-ca-file=${CA_CERT_BUNDLE_PATH}" fi # Network plugin - if [[ -n "${NETWORK_PROVIDER:-}" ]]; then - if [[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then + if [[ -n "${NETWORK_PROVIDER:-}" || -n "${NETWORK_POLICY_PROVIDER:-}" ]]; then + if [[ "${NETWORK_PROVIDER:-}" == "cni" || "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then flags+=" --cni-bin-dir=/home/kubernetes/bin" else flags+=" --network-plugin-dir=/home/kubernetes/bin" fi - flags+=" --network-plugin=${NETWORK_PROVIDER}" + if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then + # Calico uses CNI always. + flags+=" --network-plugin=cni" + else + # Otherwise use the configured value. + flags+=" --network-plugin=${NETWORK_PROVIDER}" + fi fi if [[ -n "${NON_MASQUERADE_CIDR:-}" ]]; then flags+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}" @@ -1419,6 +1425,10 @@ function start-kube-addons { fi if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then setup-addon-manifests "addons" "calico-policy-controller" + + # Replace the cluster cidr. + local -r calico_file="${dst_dir}/calico-policy-controller/calico-node.yaml" + sed -i -e "s@__CLUSTER_CIDR__@${CLUSTER_IP_RANGE}@g" "${calico_file}" fi if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then setup-addon-manifests "addons" "storage-class/gce" diff --git a/cluster/saltbase/salt/calico/10-calico.conf b/cluster/saltbase/salt/calico/10-calico.conf deleted file mode 100644 index 042dfcaa615..00000000000 --- a/cluster/saltbase/salt/calico/10-calico.conf +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "k8s-pod-network", - "type": "calico", - "etcd_authority": "10.0.0.17:6666", - "log_level": "info", - "ipam": { - "type": "host-local", - "subnet": "usePodCidr" - }, - "policy": { - "type": "k8s" - }, - "kubernetes": { - "kubeconfig": "/var/lib/kubelet/kubeconfig" - } -} diff --git a/cluster/saltbase/salt/calico/calico-node.manifest b/cluster/saltbase/salt/calico/calico-node.manifest deleted file mode 100644 index a58de672dfb..00000000000 --- a/cluster/saltbase/salt/calico/calico-node.manifest +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: calico-node - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - k8s-app: calico-node -spec: - hostNetwork: true - containers: - - name: calico-node - image: quay.io/calico/node:v0.20.0 - env: - - name: ETCD_ENDPOINTS - value: "http://10.0.0.17:6666" - - name: CALICO_NETWORKING - value: "false" - securityContext: - privileged: true - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /var/log/calico - name: var-log-calico - readOnly: false - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - volumes: - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-log-calico - hostPath: - path: /var/log/calico diff --git a/cluster/saltbase/salt/calico/master.sls b/cluster/saltbase/salt/calico/master.sls deleted file mode 100644 index 79ac253828d..00000000000 --- a/cluster/saltbase/salt/calico/master.sls +++ /dev/null @@ -1,19 +0,0 @@ -{% if pillar.get('network_policy_provider', '').lower() == 'calico' %} - -calico-policy-controller: - file.managed: - - name: /etc/kubernetes/manifests/calico-policy-controller.manifest - - source: salt://calico/calico-policy-controller.manifest - - template: jinja - - user: root - - group: root - - mode: 644 - - makedirs: true - - dir_mode: 755 - - context: - cpurequest: '20m' - - require: - - service: docker - - service: kubelet - -{% endif -%} diff --git a/cluster/saltbase/salt/calico/node.sls b/cluster/saltbase/salt/calico/node.sls index 60db9a3b2dd..dca8a371dae 100644 --- a/cluster/saltbase/salt/calico/node.sls +++ b/cluster/saltbase/salt/calico/node.sls @@ -1,37 +1,5 @@ {% if pillar.get('network_policy_provider', '').lower() == 'calico' %} -calico-node: - file.managed: - - name: /etc/kubernetes/manifests/calico-node.manifest - - source: salt://calico/calico-node.manifest - - template: jinja - - user: root - - group: root - - mode: 644 - - makedirs: true - - dir_mode: 755 - - require: - - kmod: ip6_tables - - kmod: xt_set - - service: docker - - service: kubelet - -calico-cni: - file.managed: - - name: /opt/cni/bin/calico - - source: https://github.com/projectcalico/calico-cni/releases/download/v1.3.1/calico - - source_hash: sha256=ac05cb9254b5aaa5822cf10325983431bd25489147f2edf9dec7e43d99c43e77 - - makedirs: True - - mode: 744 - -calico-cni-config: - file.managed: - - name: /etc/cni/net.d/10-calico.conf - - source: salt://calico/10-calico.conf - - makedirs: True - - mode: 644 - - template: jinja - ip6_tables: kmod.present diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index 1df05d53410..7ca7e1cfe76 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -140,7 +140,7 @@ {% elif pillar.get('network_provider', '').lower() == 'cni' %} {% set network_plugin = "--network-plugin=cni --network-plugin-dir=/etc/cni/net.d/" %} {%elif pillar.get('network_policy_provider', '').lower() == 'calico' and grains['roles'][0] != 'kubernetes-master' -%} - {% set network_plugin = "--network-plugin=cni --network-plugin-dir=/etc/cni/net.d/" %} + {% set network_plugin = "--network-plugin=cni --network-plugin-dir=/etc/cni/net.d/ --cni-bin-dir=/home/kubernetes/bin/" %} {% elif pillar.get('network_provider', '').lower() == 'kubenet' %} {% set network_plugin = "--network-plugin=kubenet" -%} {% endif -%} diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index 342c634cced..3cf3b787502 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -84,6 +84,3 @@ base: {% if pillar.get('enable_rescheduler', '').lower() == 'true' %} - rescheduler {% endif %} -{% if pillar.get('network_policy_provider', '').lower() == 'calico' %} - - calico.master -{% endif %}