Merge pull request #80536 from lzang/policy

Upgrade Calico to 3.7.4

cluster/addons/calico-policy-controller/bgppeers-crd.yaml

@@ -0,0 +1,15 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: bgppeers.crd.projectcalico.org
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: BGPPeer
+    plural: bgppeers
+    singular: bgppeer

cluster/addons/calico-policy-controller/blockaffinity-crd.yaml

@@ -0,0 +1,15 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: blockaffinities.crd.projectcalico.org
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: BlockAffinity
+    plural: blockaffinities
+    singular: blockaffinity

cluster/addons/calico-policy-controller/calico-clusterrole.yaml

@@ -7,84 +7,129 @@ metadata:
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
 rules:
+  # The CNI plugin needs to get pods, nodes, and namespaces.
   - apiGroups: [""]
     resources:
+      - pods
+      - nodes
       - namespaces
-      - serviceaccounts
     verbs:
       - get
-      - list
-      - watch
   - apiGroups: [""]
     resources:
       - endpoints
-    verbs:
-      - get
-  - apiGroups: [""]
-    resources:
       - services
     verbs:
-      - get
+      # Used to discover service IPs for advertisement.
-  - apiGroups: [""]
-    resources:
-      - pods/status
-    verbs:
-      - update
-      - patch
-  - apiGroups: [""]
-    resources:
-      - nodes
-    verbs:
-      - get
-      - list
-      - update
       - watch
+      - list
+      # Used to discover Typhas.
+      - get
+  - apiGroups: [""]
+    resources:
+      - nodes/status
+    verbs:
+      # Needed for clearing NodeNetworkUnavailable flag.
+      - patch
+      # Calico stores some configuration information in node annotations.
+      - update
+  # Watch for changes to Kubernetes NetworkPolicies.
   - apiGroups: ["networking.k8s.io"]
     resources:
       - networkpolicies
     verbs:
       - watch
       - list
-  - apiGroups: ["crd.projectcalico.org"]
+  # Used by Calico for policy information.
-    resources:
-      - felixconfigurations
-      - bgppeers
-      - bgpconfigurations
-      - ippools
-      - globalnetworkpolicies
-      - globalnetworksets
-      - networkpolicies
-      - clusterinformations
-      - hostendpoints
-    verbs:
-      - create
-      - get
-      - list
-      - update
-      - watch
-  # Used in Calico v2.6 only - can be removed after upgrade.
   - apiGroups: [""]
     resources:
       - pods
+      - namespaces
+      - serviceaccounts
     verbs:
-      - get
       - list
       - watch
+  # The CNI plugin patches pods/status.
+  - apiGroups: [""]
+    resources:
+      - pods/status
+    verbs:
       - patch
+  # Calico monitors various CRDs for config.
+  # Note: Though we are not using ipam from calico, calico node still needs those permission
+  #       to boot.
   - apiGroups: ["crd.projectcalico.org"]
     resources:
       - globalfelixconfigs
+      - felixconfigurations
+      - bgppeers
       - globalbgpconfigs
+      - bgpconfigurations
+      - ippools
+      - ipamblocks
+      - globalnetworkpolicies
+      - globalnetworksets
+      - networkpolicies
+      - networksets
+      - clusterinformations
+      - hostendpoints
+    verbs:
+      - get
+      - list
+      - watch
+  # Calico must create and update some CRDs on startup.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - ippools
+      - felixconfigurations
+      - clusterinformations
     verbs:
       - create
-      - get
-      - list
       - update
-      - watch
+  # Calico stores some configuration information on the node.
-  - apiGroups: ["extensions"]
+  - apiGroups: [""]
     resources:
-      - networkpolicies
+      - nodes
     verbs:
       - get
       - list
       - watch
+  # These permissions are only required for upgrade from v2.6, and can
+  # be removed after upgrade or on fresh installations.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - bgpconfigurations
+      - bgppeers
+    verbs:
+      - create
+      - update
+  # These permissions are required for Calico CNI to perform IPAM allocations.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - blockaffinities
+      - ipamblocks
+      - ipamhandles
+    verbs:
+      - get
+      - list
+      - create
+      - update
+      - delete
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - ipamconfigs
+    verbs:
+      - get
+  # Block affinities must also be watchable by confd for route aggregation.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - blockaffinities
+    verbs:
+      - watch
+  # The Calico IPAM migration needs to get daemonsets. These permissions can be
+  # removed if not upgrading from an installation using host-local IPAM.
+  - apiGroups: ["apps"]
+    resources:
+      - daemonsets
+    verbs:
+      - get

cluster/addons/calico-policy-controller/calico-node-daemonset.yaml

@@ -28,12 +28,60 @@ spec:
       # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
       # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
       terminationGracePeriodSeconds: 0
+      initContainers:
+        - name: install-cni
+          image: gcr.io/projectcalico-org/cni:v3.7.4
+          command: ["/install-cni.sh"]
+          env:
+            - name: CNI_CONF_NAME
+              value: "10-calico.conflist"
+            - name: CNI_NETWORK_CONFIG
+              value: |-
+                {
+                  "name": "k8s-pod-network",
+                  "cniVersion": "0.3.0",
+                  "plugins": [
+                    {
+                      "type": "calico",
+                      "log_level": "info",
+                      "datastore_type": "kubernetes",
+                      "nodename": "__KUBERNETES_NODE_NAME__",
+                      "ipam": {
+                        "type": "host-local",
+                        "subnet": "usePodCidr"
+                      },
+                      "policy": {
+                        "type": "k8s"
+                      },
+                      "kubernetes": {
+                        "kubeconfig": "__KUBECONFIG_FILEPATH__"
+                      }
+                    },
+                    {
+                      "type": "portmap",
+                      "capabilities": {"portMappings": true},
+                      "snat": true
+                    }
+                  ]
+                }
+            - name: KUBERNETES_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            # Prevents the container from sleeping forever.
+            - name: SLEEP
+              value: "false"
+          volumeMounts:
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
       containers:
         # Runs calico/node container on each Kubernetes node.  This
         # container programs network policy and routes on each
         # host.
         - name: calico-node
-          image: gcr.io/projectcalico-org/node:v3.3.1
+          image: gcr.io/projectcalico-org/node:v3.7.4
           env:
             - name: CALICO_DISABLE_FILE_LOGGING
               value: "true"
@@ -57,6 +105,8 @@ spec:
               value: "0"
             - name: FELIX_TYPHAK8SSERVICENAME
               value: "calico-typha"
+            - name: USE_POD_CIDR
+              value: "true"
             - name: IP
               value: ""
             - name: NO_DEFAULT_POOLS
@@ -96,52 +146,9 @@ spec:
             - mountPath: /var/lib/calico
               name: var-lib-calico
               readOnly: false
-        # This container installs the Calico CNI binaries
+            - mountPath: /run/xtables.lock
-        # and CNI network config file on each node.
+              name: xtables-lock
-        - name: install-cni
+              readOnly: false
-          image: gcr.io/projectcalico-org/cni:v3.3.1
-          command: ["/install-cni.sh"]
-          env:
-            - name: CNI_CONF_NAME
-              value: "10-calico.conflist"
-            - name: CNI_NETWORK_CONFIG
-              value: |-
-                {
-                  "name": "k8s-pod-network",
-                  "cniVersion": "0.3.0",
-                  "plugins": [
-                    {
-                      "type": "calico",
-                      "log_level": "info",
-                      "datastore_type": "kubernetes",
-                      "nodename": "__KUBERNETES_NODE_NAME__",
-                      "ipam": {
-                        "type": "host-local",
-                        "subnet": "usePodCidr"
-                      },
-                      "policy": {
-                        "type": "k8s"
-                      },
-                      "kubernetes": {
-                        "kubeconfig": "__KUBECONFIG_FILEPATH__"
-                      }
-                    },
-                    {
-                      "type": "portmap",
-                      "capabilities": {"portMappings": true},
-                      "snat": true
-                    }
-                  ]
-                }
-            - name: KUBERNETES_NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-          volumeMounts:
-            - mountPath: /host/opt/cni/bin
-              name: cni-bin-dir
-            - mountPath: /host/etc/cni/net.d
-              name: cni-net-dir
       volumes:
         # Used to ensure proper kmods are installed.
         - name: lib-modules
@@ -165,6 +172,10 @@ spec:
         - name: var-lib-calico
           hostPath:
             path: /var/lib/calico
+        - name: xtables-lock
+          hostPath:
+            path: /run/xtables.lock
+            type: FileOrCreate
       tolerations:
         # Make sure calico/node gets scheduled on all nodes.
         - effect: NoSchedule

cluster/addons/calico-policy-controller/ipamblock-crd.yaml

@@ -0,0 +1,15 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: ipamblocks.crd.projectcalico.org
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: IPAMBlock
+    plural: ipamblocks
+    singular: ipamblock

cluster/addons/calico-policy-controller/ipamconfig-crd.yaml

@@ -0,0 +1,15 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: ipamconfigs.crd.projectcalico.org
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: IPAMConfig
+    plural: ipamconfigs
+    singular: ipamconfig

cluster/addons/calico-policy-controller/ipamhandle-crd.yaml

@@ -0,0 +1,15 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: ipamhandles.crd.projectcalico.org
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: IPAMHandle
+    plural: ipamhandles
+    singular: ipamhandle

cluster/addons/calico-policy-controller/networkset-crd.yaml

@@ -0,0 +1,15 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: networksets.crd.projectcalico.org
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  scope: Namespaced
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: NetworkSet
+    plural: networksets
+    singular: networkset

cluster/addons/calico-policy-controller/typha-deployment.yaml

@@ -26,7 +26,7 @@ spec:
       hostNetwork: true
       serviceAccountName: calico
       containers:
-      - image: gcr.io/projectcalico-org/typha:v3.3.1
+      - image: gcr.io/projectcalico-org/typha:v3.7.4
         name: calico-typha
         ports:
         - containerPort: 5473
@@ -53,6 +53,8 @@ spec:
             value: "1"
           - name: TYPHA_HEALTHENABLED
             value: "true"
+          - name: USE_POD_CIDR
+            value: "true"
         volumeMounts:
         - mountPath: /etc/calico
           name: etc-calico