Merge pull request #29037 from matthewdupre/calico-policy-deploy

Automatic merge from submit-queue

Add support for kube-up.sh to deploy Calico network policy to GCI masters

Also remove requirement for calicoctl from Debian / salt installed nodes and clean it up a little by deploying calico-node with a manifest rather than calicoctl.  This also makes it more reliable by retrying properly.

How to use:
```
make quick-release
NETWORK_POLICY_PROVIDER=calico cluster/kube-up.sh
```

One place where I was uncertain:
- CPU allocations (on the master particularly, where there's very little spare capacity).  I took some from etcd, but if there's a better way to decide this, I'm happy to change it.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.kubernetes.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.kubernetes.io/reviews/kubernetes/kubernetes/29037)
<!-- Reviewable:end -->

cluster/addons/calico-policy-controller/MAINTAINERS.md

@@ -0,0 +1,6 @@
+# Maintainers
+
+Matt Dupre <matt@projectcalico.org>, Casey Davenport <casey@tigera.io> and committers to the https://github.com/projectcalico/k8s-policy repository.
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/calico-policy-controller/MAINTAINERS.md?pixel)]()

cluster/addons/calico-policy-controller/README.md

@@ -0,0 +1,11 @@
+# Calico Policy Controller
+==============
+
+Calico Policy Controller is an implementation of the Kubernetes network policy API.
+
+Learn more at:
+- https://github.com/projectcalico/k8s-policy
+- http://kubernetes.io/docs/user-guide/networkpolicies/
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/calico-policy-controller/README.md?pixel)]()

cluster/addons/calico-policy-controller/calico-etcd-petset.yaml

@@ -0,0 +1,43 @@
+apiVersion: "apps/v1alpha1"
+kind: PetSet
+metadata:
+  name: calico-etcd
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    k8s-app: calico-etcd
+spec:
+  serviceName: calico-etcd
+  replicas: 1
+  template:
+    metadata:
+      annotations:
+        pod.alpha.kubernetes.io/initialized: "true"
+      labels:
+        kubernetes.io/cluster-service: "true"
+        k8s-app: calico-etcd
+    spec:
+      hostNetwork: true
+      containers:
+        - name: calico-etcd
+          image: gcr.io/google_containers/etcd:2.2.1
+          env:
+            - name: CALICO_ETCD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+          command: ["/bin/sh","-c"]
+          args: ["/usr/local/bin/etcd --name=calico --data-dir=/var/etcd/calico-data --advertise-client-urls=http://$CALICO_ETCD_IP:6666 --listen-client-urls=http://0.0.0.0:6666 --listen-peer-urls=http://0.0.0.0:6667"]
+          volumeMounts:
+            - name: var-etcd
+              mountPath: /var/etcd
+  volumeClaimTemplates:
+    - metadata:
+        name: var-etcd
+        annotations:
+          volume.alpha.kubernetes.io/storage-class: anything
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: 1Gi

cluster/addons/calico-policy-controller/calico-etcd-service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: calico-etcd
+    kubernetes.io/cluster-service: "true"
+  name: calico-etcd
+  namespace: kube-system
+spec:
+  clusterIP: 10.0.0.17
+  ports:
+    - port: 6666
+  selector:
+    k8s-app: calico-etcd

cluster/addons/calico-policy-controller/calico-policy-controller.yaml

@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: calico-policy-controller
+  namespace: kube-system
+  labels:
+    k8s-app: calico-policy
+    kubernetes.io/cluster-service: "true"
+spec:
+  replicas: 1
+  selector:
+    k8s-app: calico-policy
+  template:
+    metadata:
+      name: calico-policy-controller
+      namespace: kube-system
+      labels:
+        kubernetes.io/cluster-service: "true"
+        k8s-app: calico-policy
+    spec:
+      hostNetwork: true
+      containers:
+        - name: calico-policy-controller
+          image: calico/kube-policy-controller:v0.2.0
+          env:
+            - name: ETCD_ENDPOINTS
+              value: "http://10.0.0.17:6666"
+            - name: K8S_API
+              value: "https://kubernetes.default:443"
+            - name: CONFIGURE_ETC_HOSTS
+              value: "true"

cluster/common.sh

@@ -568,6 +568,7 @@ CA_CERT: $(yaml-quote ${CA_CERT_BASE64:-})
 KUBELET_CERT: $(yaml-quote ${KUBELET_CERT_BASE64:-})
 KUBELET_KEY: $(yaml-quote ${KUBELET_KEY_BASE64:-})
 NETWORK_PROVIDER: $(yaml-quote ${NETWORK_PROVIDER:-})
+NETWORK_POLICY_PROVIDER: $(yaml-quote ${NETWORK_POLICY_PROVIDER:-})
 PREPULL_E2E_IMAGES: $(yaml-quote ${PREPULL_E2E_IMAGES:-})
 HAIRPIN_MODE: $(yaml-quote ${HAIRPIN_MODE:-})
 OPENCONTRAIL_TAG: $(yaml-quote ${OPENCONTRAIL_TAG:-})

cluster/gce/gci/configure-helper.sh

@@ -945,6 +945,9 @@ function start-kube-addons {
   if echo "${ADMISSION_CONTROL:-}" | grep -q "LimitRanger"; then
     setup-addon-manifests "admission-controls" "limit-range"
   fi
+  if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
+    setup-addon-manifests "addons" "calico-policy-controller"
+  fi
 
   # Place addon manager pod manifest.
   cp "${src_dir}/kube-addon-manager.yaml" /etc/kubernetes/manifests

cluster/saltbase/salt/calico/10-calico.conf

@@ -1,7 +1,7 @@
 {
     "name": "k8s-pod-network",
     "type": "calico",
-    "etcd_authority": "{{ grains.api_servers }}:6666",
+    "etcd_authority": "10.0.0.17:6666",
     "log_level": "info",
     "ipam": {
         "type": "host-local",

cluster/saltbase/salt/calico/calico-node.manifest

@@ -0,0 +1,40 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: calico-node
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    k8s-app: calico-node
+spec:
+  hostNetwork: true
+  containers:
+    - name: calico-node
+      image: quay.io/calico/node:v0.20.0
+      env:
+        - name: ETCD_ENDPOINTS
+          value: "http://10.0.0.17:6666"
+        - name: CALICO_NETWORKING
+          value: "false"
+      securityContext:
+        privileged: true
+      volumeMounts:
+        - mountPath: /lib/modules
+          name: lib-modules
+          readOnly: true
+        - mountPath: /var/log/calico
+          name: var-log-calico
+          readOnly: false
+        - mountPath: /var/run/calico
+          name: var-run-calico
+          readOnly: false
+  volumes:
+    - name: lib-modules
+      hostPath:
+        path: /lib/modules
+    - name: var-run-calico
+      hostPath:
+        path: /var/run/calico
+    - name: var-log-calico
+      hostPath:
+        path: /var/log/calico

cluster/saltbase/salt/calico/calico-policy-controller.manifest

@@ -1,36 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: calico-policy-controller
-  namespace: kube-system
-  labels:
-    kubernetes.io/cluster-service: "true"
-    k8s-app: calico-policy
-spec:
-  hostNetwork: true
-  containers:
-    - name: policy-controller
-      image: calico/kube-policy-controller:v0.2.0
-      env:
-      - name: ETCD_AUTHORITY
-        value: "127.0.0.1:6666"
-      - name: K8S_API
-        value: "http://127.0.0.1:8080"
-    - name: calico-etcd
-      image: gcr.io/google_containers/etcd:2.2.1
-      command:
-        - /usr/local/bin/etcd
-        - --name=calico
-        - --data-dir=/var/etcd/calico-data
-        - --advertise-client-urls=http://{{ grains.id }}:6666
-        - --listen-client-urls=http://0.0.0.0:6666
-        - --listen-peer-urls=http://0.0.0.0:6667
-        - --initial-advertise-peer-urls=http://{{ grains.id }}:6667
-        - --initial-cluster=calico=http://{{ grains.id }}:6667
-      volumeMounts:
-        - name: varetcd
-          mountPath: /var/etcd
-  volumes:
-    - name: varetcd
-      hostPath:
-        path: /var/calico/etcd

cluster/saltbase/salt/calico/node.sls

@@ -1,30 +1,25 @@
 {% if pillar.get('network_policy_provider', '').lower() == 'calico' %}
 
-calicoctl:
-  file.managed:
-    - name: /usr/bin/calicoctl
-    - source: https://github.com/projectcalico/calico-docker/releases/download/v0.19.0/calicoctl
-    - source_hash: sha256=6db00c94619e82d878d348c4e1791f8d2f0db59075f6c8e430fefae297c54d96
-    - makedirs: True
-    - mode: 744
-
 calico-node:
-  cmd.run:
+  file.managed:
-    - name: calicoctl node
+    - name: /etc/kubernetes/manifests/calico-node.manifest
-    - unless: docker ps | grep calico-node
+    - source: salt://calico/calico-node.manifest
-    - env:
+    - template: jinja
-      - ETCD_AUTHORITY: "{{ grains.api_servers }}:6666"
+    - user: root
-      - CALICO_NETWORKING: "false"
+    - group: root
+    - mode: 644
+    - makedirs: true
+    - dir_mode: 755
     - require:
       - kmod: ip6_tables
       - kmod: xt_set
       - service: docker
-      - file: calicoctl
+      - service: kubelet
 
 calico-cni:
   file.managed:
     - name: /opt/cni/bin/calico
-    - source: https://github.com/projectcalico/calico-cni/releases/download/v1.3.1/calico 
+    - source: https://github.com/projectcalico/calico-cni/releases/download/v1.3.1/calico
     - source_hash: sha256=ac05cb9254b5aaa5822cf10325983431bd25489147f2edf9dec7e43d99c43e77
     - makedirs: True
     - mode: 744