Update canal and calico to 3.1.1

cluster.yml

@@ -83,6 +83,12 @@ services:
 #   plugin: flannel
 #   options:
 #     flannel_iface: eth1
+# To specify flannel interface for canal plugin, you can use the 'canal_iface' option:
+# network:
+#   plugin: canal
+#   options:
+#     canal_iface: eth1
+
 
 network:
   plugin: flannel

cluster/defaults.go

@@ -169,6 +169,9 @@ func (c *Cluster) setClusterNetworkDefaults() {
 	if c.Network.FlannelNetworkProvider != nil {
 		networkPluginConfigDefaultsMap[FlannelIface] = c.Network.FlannelNetworkProvider.Iface
 	}
+	if c.Network.CanalNetworkProvider != nil {
+		networkPluginConfigDefaultsMap[CanalIface] = c.Network.CanalNetworkProvider.Iface
+	}
 	for k, v := range networkPluginConfigDefaultsMap {
 		setDefaultIfEmptyMapValue(c.Network.Options, k, v)
 	}

cluster/network.go

@@ -56,6 +56,7 @@ const (
 	CanalNodeImage     = "canal_node_image"
 	CanalCNIImage      = "canal_cni_image"
 	CanalFlannelImage  = "canal_flannel_image"
+	CanalIface         = "canal_iface"
 
 	WeaveNetworkPlugin = "weave"
 	WeaveImage         = "weave_node_image"
@@ -94,6 +95,7 @@ const (
 	Calicoctl = "Calicoctl"
 
 	FlannelInterface = "FlannelInterface"
+	CanalInterface   = "CanalInterface"
 	RBACConfig       = "RBACConfig"
 )
 
@@ -172,6 +174,7 @@ func (c *Cluster) doCanalDeploy(ctx context.Context) error {
 		CNIImage:        c.SystemImages.CanalCNI,
 		CanalFlannelImg: c.SystemImages.CanalFlannel,
 		RBACConfig:      c.Authorization.Mode,
+		CanalInterface:  c.Network.Options[CanalIface],
 	}
 	pluginYaml, err := c.getNetworkPluginManifest(canalConfig)
 	if err != nil {

cluster/plan.go

@@ -322,6 +322,7 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string) v3.Pr
 		"/etc/cni:/etc/cni:rw,z",
 		"/opt/cni:/opt/cni:rw,z",
 		fmt.Sprintf("%s:/var/lib/cni:z", path.Join(prefixPath, "/var/lib/cni")),
+		"/var/lib/calico:/var/lib/calico:z",
 		"/etc/resolv.conf:/etc/resolv.conf",
 		"/sys:/sys:rprivate",
 		host.DockerInfo.DockerRootDir + ":" + host.DockerInfo.DockerRootDir + ":rw,rslave,z",

templates/calico.go

@@ -3,7 +3,7 @@ package templates
 const CalicoTemplate = `
 {{if eq .RBACConfig "rbac"}}
 ## start rbac here
----
+
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
@@ -54,6 +54,12 @@ rules:
       - get
       - list
       - watch
+  - apiGroups: ["networking.k8s.io"]
+    resources:
+      - networkpolicies
+    verbs:
+      - watch
+      - list
   - apiGroups: ["crd.projectcalico.org"]
     resources:
       - globalfelixconfigs
@@ -63,17 +69,17 @@ rules:
       - bgpconfigurations
       - ippools
       - globalnetworkpolicies
+      - globalnetworksets
       - networkpolicies
       - clusterinformations
+      - hostendpoints
     verbs:
       - create
       - get
       - list
       - update
       - watch
-
 ---
-
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
 metadata:
@@ -91,6 +97,7 @@ subjects:
   name: system:nodes
 {{end}}
 ## end rbac here
+
 ---
 kind: ConfigMap
 apiVersion: v1
@@ -160,22 +167,29 @@ spec:
       labels:
         k8s-app: calico-node
       annotations:
+        # This, along with the CriticalAddonsOnly toleration below,
+        # marks the pod as a critical add-on, ensuring it gets
+        # priority scheduling and that its resources are reserved
+        # if it ever gets evicted.
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       hostNetwork: true
-      serviceAccountName: calico-node
-      terminationGracePeriodSeconds: 0
       tolerations:
-        - key: "dedicated"
+        # Make sure calico/node gets scheduled on all nodes.
-          value: "master"
+        - effect: NoSchedule
-          effect: "NoSchedule"
+          operator: Exists
-        - key: "CriticalAddonsOnly"
+        # Mark the pod as a critical add-on for rescheduling.
-          operator: "Exists"
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
         - key: "node-role.kubernetes.io/master"
           operator: "Exists"
         - key: "node-role.kubernetes.io/etcd"
           operator: "Exists"
           effect: "NoExecute"
+      serviceAccountName: calico-node
+      terminationGracePeriodSeconds: 0
       containers:
         # Runs calico/node container on each Kubernetes node.  This
         # container programs network policy and routes on each
@@ -186,37 +200,45 @@ spec:
             # Use Kubernetes API as the backing datastore.
             - name: DATASTORE_TYPE
               value: "kubernetes"
-            # Wait for the datastore.
+            # Enable felix info logging.
-            - name: WAIT_FOR_DATASTORE
+            - name: FELIX_LOGSEVERITYSCREEN
-              value: "true"
+              value: "info"
             # Cluster type to identify the deployment type
             - name: CLUSTER_TYPE
               value: "k8s,bgp"
-            # Disable file logging so "kubectl logs" works.
+            # Disable file logging so kubectl logs works.
             - name: CALICO_DISABLE_FILE_LOGGING
               value: "true"
             # Set Felix endpoint to host default action to ACCEPT.
             - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
               value: "ACCEPT"
-            # Configure the IP Pool from which Pod IPs will be chosen.
+            # Disable IPV6 on Kubernetes.
-            - name: CALICO_IPV4POOL_CIDR
-              value: "{{.ClusterCIDR}}"
-            - name: CALICO_IPV4POOL_IPIP
-              value: "Always"
-            # Disable IPv6 on Kubernetes.
             - name: FELIX_IPV6SUPPORT
               value: "false"
-            # Set Felix logging to "info"
-            - name: FELIX_LOGSEVERITYSCREEN
-              value: "info"
             # Set MTU for tunnel device used if ipip is enabled
             - name: FELIX_IPINIPMTU
               value: "1440"
-            # Auto-detect the BGP IP address.
+            # Wait for the datastore.
-            - name: IP
+            - name: WAIT_FOR_DATASTORE
-              value: ""
-            - name: FELIX_HEALTHENABLED
               value: "true"
+            # The default IPv4 pool to create on startup if none exists. Pod IPs will be
+            # chosen from this range. Changing this value after installation will have
+            # no effect. This should fall within --cluster-cidr.
+            - name: CALICO_IPV4POOL_CIDR
+              value: "{{.ClusterCIDR}}"
+            # Enable IPIP
+            - name: CALICO_IPV4POOL_IPIP
+              value: "Always"
+            # Enable IP-in-IP within Felix.
+            - name: FELIX_IPINIPENABLED
+              value: "true"
+            # Typha support: controlled by the ConfigMap.
+            - name: FELIX_TYPHAK8SSERVICENAME
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: typha_service_name
+            # Set based on the k8s node name.
             - name: NODENAME
               valueFrom:
                 fieldRef:
@@ -250,8 +272,9 @@ spec:
             - mountPath: /var/run/calico
               name: var-run-calico
               readOnly: false
-            - mountPath: /etc/kubernetes
+            - mountPath: /var/lib/calico
-              name: etc-kubernetes
+              name: var-lib-calico
+              readOnly: false
         # This container installs the Calico CNI binaries
         # and CNI network config file on each node.
         - name: install-cni
@@ -277,8 +300,6 @@ spec:
               name: cni-bin-dir
             - mountPath: /host/etc/cni/net.d
               name: cni-net-dir
-            - mountPath: /etc/kubernetes
-              name: etc-kubernetes
       volumes:
         # Used by calico/node.
         - name: lib-modules
@@ -287,6 +308,9 @@ spec:
         - name: var-run-calico
           hostPath:
             path: /var/run/calico
+        - name: var-lib-calico
+          hostPath:
+            path: /var/lib/calico
         # Used to install CNI.
         - name: cni-bin-dir
           hostPath:
@@ -294,10 +318,9 @@ spec:
         - name: cni-net-dir
           hostPath:
             path: /etc/cni/net.d
-        - name: etc-kubernetes
-          hostPath:
-            path: /etc/kubernetes
 
+# Create all the CustomResourceDefinitions needed for
+# Calico policy and networking mode.
 ---
 
 apiVersion: apiextensions.k8s.io/v1beta1
@@ -363,6 +386,22 @@ spec:
 
 ---
 
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico HostEndpoints
+kind: CustomResourceDefinition
+metadata:
+  name: hostendpoints.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: HostEndpoint
+    plural: hostendpoints
+    singular: hostendpoint
+
+---
+
 apiVersion: apiextensions.k8s.io/v1beta1
 description: Calico Cluster Information
 kind: CustomResourceDefinition
@@ -395,6 +434,22 @@ spec:
 
 ---
 
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Global Network Sets
+kind: CustomResourceDefinition
+metadata:
+  name: globalnetworksets.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: GlobalNetworkSet
+    plural: globalnetworksets
+    singular: globalnetworkset
+
+---
+
 apiVersion: apiextensions.k8s.io/v1beta1
 description: Calico Network Policies
 kind: CustomResourceDefinition

templates/canal.go

@@ -2,9 +2,7 @@ package templates
 
 const CanalTemplate = `
 {{if eq .RBACConfig "rbac"}}
----
 # Calico Roles
-# Pulled from https://docs.projectcalico.org/v2.5/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
@@ -29,6 +27,17 @@ rules:
       - get
       - list
       - watch
+      - patch
+  - apiGroups: [""]
+    resources:
+      - services
+    verbs:
+      - get
+  - apiGroups: [""]
+    resources:
+      - endpoints
+    verbs:
+      - get
   - apiGroups: [""]
     resources:
       - nodes
@@ -37,7 +46,7 @@ rules:
       - list
       - update
       - watch
-  - apiGroups: ["extensions"]
+  - apiGroups: ["networking.k8s.io"]
     resources:
       - networkpolicies
     verbs:
@@ -47,10 +56,16 @@ rules:
   - apiGroups: ["crd.projectcalico.org"]
     resources:
       - globalfelixconfigs
+      - felixconfigurations
       - bgppeers
       - globalbgpconfigs
+      - bgpconfigurations
       - ippools
       - globalnetworkpolicies
+      - networkpolicies
+      - clusterinformations
+      - hostendpoints
+      - globalnetworksets
     verbs:
       - create
       - get
@@ -117,10 +132,18 @@ subjects:
 - kind: ServiceAccount
   name: canal
   namespace: kube-system
-
+- apiGroup: rbac.authorization.k8s.io
-## end rbac
+  kind: Group
+  name: system:nodes
 {{end}}
 
+# Canal Version v3.1.1
+# https://docs.projectcalico.org/v3.1/releases#v3.1.1
+# This manifest includes the following component versions:
+#   calico/node:v3.1.1
+#   calico/cni:v3.1.1
+#   coreos/flannel:v0.9.1
+
 ---
 # This ConfigMap can be used to configure a self-hosted Canal installation.
 kind: ConfigMap
@@ -132,7 +155,7 @@ data:
   # The interface used by canal for host <-> host communication.
   # If left blank, then the interface is chosen using the node's
   # default route.
-  canal_iface: ""
+  canal_iface: "{{.CanalInterface}}"
 
   # Whether or not to masquerade traffic to destinations not within
   # the pod network.
@@ -141,7 +164,7 @@ data:
   # The CNI network configuration to install on each node.
   cni_network_config: |-
     {
-        "name": "rke-pod-network",
+      "name": "k8s-pod-network",
       "cniVersion": "0.3.0",
       "plugins": [
         {
@@ -155,19 +178,17 @@ data:
           },
           "policy": {
             "type": "k8s",
-                    "k8s_api_root": "{{.APIRoot}}",
+            "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__"
-                    "k8s_client_certificate": "{{.ClientCertPath}}",
-                    "k8s_client_key": "{{.ClientKeyPath}}",
-                    "k8s_certificate_authority": "{{.ClientCAPath}}"
           },
           "kubernetes": {
+            "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__",
             "kubeconfig": "{{.KubeCfg}}"
           }
         },
         {
           "type": "portmap",
-                "capabilities": {"portMappings": true},
+          "snat": true,
-                "snat": true
+          "capabilities": {"portMappings": true}
         }
       ]
     }
@@ -197,6 +218,10 @@ spec:
   selector:
     matchLabels:
       k8s-app: canal
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
   template:
     metadata:
       labels:
@@ -207,21 +232,19 @@ spec:
       hostNetwork: true
       serviceAccountName: canal
       tolerations:
-        # this taint is set by all kubelets running '--cloud-provider=external'
+        # Tolerate this effect so the pods will be schedulable at all times
-        # so we should tolerate it to schedule the canal pods
+        - effect: NoSchedule
-        - key: node.cloudprovider.kubernetes.io/uninitialized
+          operator: Exists
-          value: "true"
+        # Mark the pod as a critical add-on for rescheduling.
-          effect: NoSchedule
+        - key: CriticalAddonsOnly
-        # Allow the pod to run on the master abd etcd.  This is required for
+          operator: Exists
-        # the master to communicate with pods.
+        - effect: NoExecute
+          operator: Exists
         - key: "node-role.kubernetes.io/master"
           operator: "Exists"
         - key: "node-role.kubernetes.io/etcd"
           operator: "Exists"
           effect: "NoExecute"
-        # Mark the pod as a critical add-on for rescheduling.
-        - key: "CriticalAddonsOnly"
-          operator: "Exists"
       # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
       # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
       terminationGracePeriodSeconds: 0
@@ -244,7 +267,7 @@ spec:
             # Cluster type to identify the deployment type
             - name: CLUSTER_TYPE
               value: "k8s,canal"
-            # Disable file logging so 'kubectl logs' works.
+            # Disable file logging so kubectl logs works.
             - name: CALICO_DISABLE_FILE_LOGGING
               value: "true"
             # Period, in seconds, at which felix re-applies all iptables state
@@ -292,8 +315,9 @@ spec:
             - mountPath: /var/run/calico
               name: var-run-calico
               readOnly: false
-            - mountPath: /etc/kubernetes
+            - mountPath: /var/lib/calico
-              name: etc-kubernetes
+              name: var-lib-calico
+              readOnly: false
         # This container installs the Calico CNI binaries
         # and CNI network config file on each node.
         - name: install-cni
@@ -317,8 +341,6 @@ spec:
               name: cni-bin-dir
             - mountPath: /host/etc/cni/net.d
               name: cni-net-dir
-            - mountPath: /etc/kubernetes
-              name: etc-kubernetes
         # This container runs flannel using the kube-subnet-mgr backend
         # for allocating subnets.
         - name: kube-flannel
@@ -358,6 +380,9 @@ spec:
         - name: var-run-calico
           hostPath:
             path: /var/run/calico
+        - name: var-lib-calico
+          hostPath:
+            path: /var/lib/calico
         # Used to install CNI.
         - name: cni-bin-dir
           hostPath:
@@ -372,44 +397,40 @@ spec:
         - name: flannel-cfg
           configMap:
             name: canal-config
-        - name: etc-kubernetes
-          hostPath:
-            path: /etc/kubernetes
-
 
 # Create all the CustomResourceDefinitions needed for
 # Calico policy-only mode.
 ---
 
 apiVersion: apiextensions.k8s.io/v1beta1
-description: Calico Global Felix Configuration
+description: Calico Felix Configuration
 kind: CustomResourceDefinition
 metadata:
-   name: globalfelixconfigs.crd.projectcalico.org
+   name: felixconfigurations.crd.projectcalico.org
 spec:
   scope: Cluster
   group: crd.projectcalico.org
   version: v1
   names:
-    kind: GlobalFelixConfig
+    kind: FelixConfiguration
-    plural: globalfelixconfigs
+    plural: felixconfigurations
-    singular: globalfelixconfig
+    singular: felixconfiguration
 
 ---
 
 apiVersion: apiextensions.k8s.io/v1beta1
-description: Calico Global BGP Configuration
+description: Calico BGP Configuration
 kind: CustomResourceDefinition
 metadata:
-  name: globalbgpconfigs.crd.projectcalico.org
+  name: bgpconfigurations.crd.projectcalico.org
 spec:
   scope: Cluster
   group: crd.projectcalico.org
   version: v1
   names:
-    kind: GlobalBGPConfig
+    kind: BGPConfiguration
-    plural: globalbgpconfigs
+    plural: bgpconfigurations
-    singular: globalbgpconfig
+    singular: bgpconfiguration
 
 ---
 
@@ -429,6 +450,22 @@ spec:
 
 ---
 
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Cluster Information
+kind: CustomResourceDefinition
+metadata:
+  name: clusterinformations.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: ClusterInformation
+    plural: clusterinformations
+    singular: clusterinformation
+
+---
+
 apiVersion: apiextensions.k8s.io/v1beta1
 description: Calico Global Network Policies
 kind: CustomResourceDefinition
@@ -445,8 +482,57 @@ spec:
 
 ---
 
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Network Policies
+kind: CustomResourceDefinition
+metadata:
+  name: networkpolicies.crd.projectcalico.org
+spec:
+  scope: Namespaced
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: NetworkPolicy
+    plural: networkpolicies
+    singular: networkpolicy
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Global Network Sets
+kind: CustomResourceDefinition
+metadata:
+  name: globalnetworksets.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: GlobalNetworkSet
+    plural: globalnetworksets
+    singular: globalnetworkset
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Host Endpoints
+kind: CustomResourceDefinition
+metadata:
+  name: hostendpoints.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: HostEndpoint
+    plural: hostendpoints
+    singular: hostendpoint
+
+---
+
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: canal
-  namespace: kube-system`
+  namespace: kube-system
+`