diff --git a/cluster.yml b/cluster.yml index a990c3d4..2a1491ec 100644 --- a/cluster.yml +++ b/cluster.yml @@ -83,6 +83,12 @@ services: # plugin: flannel # options: # flannel_iface: eth1 +# To specify flannel interface for canal plugin, you can use the 'canal_iface' option: +# network: +# plugin: canal +# options: +# canal_iface: eth1 + network: plugin: flannel diff --git a/cluster/defaults.go b/cluster/defaults.go index 89f8725a..246b5729 100644 --- a/cluster/defaults.go +++ b/cluster/defaults.go @@ -169,6 +169,9 @@ func (c *Cluster) setClusterNetworkDefaults() { if c.Network.FlannelNetworkProvider != nil { networkPluginConfigDefaultsMap[FlannelIface] = c.Network.FlannelNetworkProvider.Iface } + if c.Network.CanalNetworkProvider != nil { + networkPluginConfigDefaultsMap[CanalIface] = c.Network.CanalNetworkProvider.Iface + } for k, v := range networkPluginConfigDefaultsMap { setDefaultIfEmptyMapValue(c.Network.Options, k, v) } diff --git a/cluster/network.go b/cluster/network.go index 6a3cd05d..5c8cdaf2 100644 --- a/cluster/network.go +++ b/cluster/network.go @@ -56,6 +56,7 @@ const ( CanalNodeImage = "canal_node_image" CanalCNIImage = "canal_cni_image" CanalFlannelImage = "canal_flannel_image" + CanalIface = "canal_iface" WeaveNetworkPlugin = "weave" WeaveImage = "weave_node_image" @@ -94,6 +95,7 @@ const ( Calicoctl = "Calicoctl" FlannelInterface = "FlannelInterface" + CanalInterface = "CanalInterface" RBACConfig = "RBACConfig" ) @@ -172,6 +174,7 @@ func (c *Cluster) doCanalDeploy(ctx context.Context) error { CNIImage: c.SystemImages.CanalCNI, CanalFlannelImg: c.SystemImages.CanalFlannel, RBACConfig: c.Authorization.Mode, + CanalInterface: c.Network.Options[CanalIface], } pluginYaml, err := c.getNetworkPluginManifest(canalConfig) if err != nil { diff --git a/cluster/plan.go b/cluster/plan.go index bdd64811..d1132f10 100644 --- a/cluster/plan.go +++ b/cluster/plan.go @@ -322,6 +322,7 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string) v3.Pr "/etc/cni:/etc/cni:rw,z", "/opt/cni:/opt/cni:rw,z", fmt.Sprintf("%s:/var/lib/cni:z", path.Join(prefixPath, "/var/lib/cni")), + "/var/lib/calico:/var/lib/calico:z", "/etc/resolv.conf:/etc/resolv.conf", "/sys:/sys:rprivate", host.DockerInfo.DockerRootDir + ":" + host.DockerInfo.DockerRootDir + ":rw,rslave,z", diff --git a/templates/calico.go b/templates/calico.go index 53f855f2..de2c5dde 100644 --- a/templates/calico.go +++ b/templates/calico.go @@ -3,7 +3,7 @@ package templates const CalicoTemplate = ` {{if eq .RBACConfig "rbac"}} ## start rbac here ---- + kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: @@ -54,6 +54,12 @@ rules: - get - list - watch + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list - apiGroups: ["crd.projectcalico.org"] resources: - globalfelixconfigs @@ -63,17 +69,17 @@ rules: - bgpconfigurations - ippools - globalnetworkpolicies + - globalnetworksets - networkpolicies - clusterinformations + - hostendpoints verbs: - create - get - list - update - watch - --- - apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: @@ -91,6 +97,7 @@ subjects: name: system:nodes {{end}} ## end rbac here + --- kind: ConfigMap apiVersion: v1 @@ -160,22 +167,29 @@ spec: labels: k8s-app: calico-node annotations: + # This, along with the CriticalAddonsOnly toleration below, + # marks the pod as a critical add-on, ensuring it gets + # priority scheduling and that its resources are reserved + # if it ever gets evicted. scheduler.alpha.kubernetes.io/critical-pod: '' spec: hostNetwork: true - serviceAccountName: calico-node - terminationGracePeriodSeconds: 0 tolerations: - - key: "dedicated" - value: "master" - effect: "NoSchedule" - - key: "CriticalAddonsOnly" - operator: "Exists" + # Make sure calico/node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists - key: "node-role.kubernetes.io/master" operator: "Exists" - key: "node-role.kubernetes.io/etcd" operator: "Exists" effect: "NoExecute" + serviceAccountName: calico-node + terminationGracePeriodSeconds: 0 containers: # Runs calico/node container on each Kubernetes node. This # container programs network policy and routes on each @@ -186,37 +200,45 @@ spec: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE value: "kubernetes" - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" + # Enable felix info logging. + - name: FELIX_LOGSEVERITYSCREEN + value: "info" # Cluster type to identify the deployment type - name: CLUSTER_TYPE value: "k8s,bgp" - # Disable file logging so "kubectl logs" works. + # Disable file logging so kubectl logs works. - name: CALICO_DISABLE_FILE_LOGGING value: "true" # Set Felix endpoint to host default action to ACCEPT. - name: FELIX_DEFAULTENDPOINTTOHOSTACTION value: "ACCEPT" - # Configure the IP Pool from which Pod IPs will be chosen. - - name: CALICO_IPV4POOL_CIDR - value: "{{.ClusterCIDR}}" - - name: CALICO_IPV4POOL_IPIP - value: "Always" - # Disable IPv6 on Kubernetes. + # Disable IPV6 on Kubernetes. - name: FELIX_IPV6SUPPORT value: "false" - # Set Felix logging to "info" - - name: FELIX_LOGSEVERITYSCREEN - value: "info" # Set MTU for tunnel device used if ipip is enabled - name: FELIX_IPINIPMTU value: "1440" - # Auto-detect the BGP IP address. - - name: IP - value: "" - - name: FELIX_HEALTHENABLED + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE value: "true" + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within --cluster-cidr. + - name: CALICO_IPV4POOL_CIDR + value: "{{.ClusterCIDR}}" + # Enable IPIP + - name: CALICO_IPV4POOL_IPIP + value: "Always" + # Enable IP-in-IP within Felix. + - name: FELIX_IPINIPENABLED + value: "true" + # Typha support: controlled by the ConfigMap. + - name: FELIX_TYPHAK8SSERVICENAME + valueFrom: + configMapKeyRef: + name: calico-config + key: typha_service_name + # Set based on the k8s node name. - name: NODENAME valueFrom: fieldRef: @@ -250,8 +272,9 @@ spec: - mountPath: /var/run/calico name: var-run-calico readOnly: false - - mountPath: /etc/kubernetes - name: etc-kubernetes + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni @@ -277,8 +300,6 @@ spec: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - mountPath: /etc/kubernetes - name: etc-kubernetes volumes: # Used by calico/node. - name: lib-modules @@ -287,6 +308,9 @@ spec: - name: var-run-calico hostPath: path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico # Used to install CNI. - name: cni-bin-dir hostPath: @@ -294,10 +318,9 @@ spec: - name: cni-net-dir hostPath: path: /etc/cni/net.d - - name: etc-kubernetes - hostPath: - path: /etc/kubernetes +# Create all the CustomResourceDefinitions needed for +# Calico policy and networking mode. --- apiVersion: apiextensions.k8s.io/v1beta1 @@ -363,6 +386,22 @@ spec: --- +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico HostEndpoints +kind: CustomResourceDefinition +metadata: + name: hostendpoints.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: HostEndpoint + plural: hostendpoints + singular: hostendpoint + +--- + apiVersion: apiextensions.k8s.io/v1beta1 description: Calico Cluster Information kind: CustomResourceDefinition @@ -395,6 +434,22 @@ spec: --- +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico Global Network Sets +kind: CustomResourceDefinition +metadata: + name: globalnetworksets.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkSet + plural: globalnetworksets + singular: globalnetworkset + +--- + apiVersion: apiextensions.k8s.io/v1beta1 description: Calico Network Policies kind: CustomResourceDefinition diff --git a/templates/canal.go b/templates/canal.go index 90fd3eca..74408ef9 100644 --- a/templates/canal.go +++ b/templates/canal.go @@ -2,9 +2,7 @@ package templates const CanalTemplate = ` {{if eq .RBACConfig "rbac"}} ---- # Calico Roles -# Pulled from https://docs.projectcalico.org/v2.5/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: @@ -29,6 +27,17 @@ rules: - get - list - watch + - patch + - apiGroups: [""] + resources: + - services + verbs: + - get + - apiGroups: [""] + resources: + - endpoints + verbs: + - get - apiGroups: [""] resources: - nodes @@ -37,7 +46,7 @@ rules: - list - update - watch - - apiGroups: ["extensions"] + - apiGroups: ["networking.k8s.io"] resources: - networkpolicies verbs: @@ -47,10 +56,16 @@ rules: - apiGroups: ["crd.projectcalico.org"] resources: - globalfelixconfigs + - felixconfigurations - bgppeers - globalbgpconfigs + - bgpconfigurations - ippools - globalnetworkpolicies + - networkpolicies + - clusterinformations + - hostendpoints + - globalnetworksets verbs: - create - get @@ -117,10 +132,18 @@ subjects: - kind: ServiceAccount name: canal namespace: kube-system - -## end rbac +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes {{end}} +# Canal Version v3.1.1 +# https://docs.projectcalico.org/v3.1/releases#v3.1.1 +# This manifest includes the following component versions: +# calico/node:v3.1.1 +# calico/cni:v3.1.1 +# coreos/flannel:v0.9.1 + --- # This ConfigMap can be used to configure a self-hosted Canal installation. kind: ConfigMap @@ -132,7 +155,7 @@ data: # The interface used by canal for host <-> host communication. # If left blank, then the interface is chosen using the node's # default route. - canal_iface: "" + canal_iface: "{{.CanalInterface}}" # Whether or not to masquerade traffic to destinations not within # the pod network. @@ -141,35 +164,33 @@ data: # The CNI network configuration to install on each node. cni_network_config: |- { - "name": "rke-pod-network", - "cniVersion": "0.3.0", - "plugins": [ - { - "type": "calico", - "log_level": "info", - "datastore_type": "kubernetes", - "nodename": "__KUBERNETES_NODE_NAME__", - "ipam": { - "type": "host-local", - "subnet": "usePodCidr" - }, - "policy": { - "type": "k8s", - "k8s_api_root": "{{.APIRoot}}", - "k8s_client_certificate": "{{.ClientCertPath}}", - "k8s_client_key": "{{.ClientKeyPath}}", - "k8s_certificate_authority": "{{.ClientCAPath}}" - }, - "kubernetes": { - "kubeconfig": "{{.KubeCfg}}" - } - }, - { - "type": "portmap", - "capabilities": {"portMappings": true}, - "snat": true - } - ] + "name": "k8s-pod-network", + "cniVersion": "0.3.0", + "plugins": [ + { + "type": "calico", + "log_level": "info", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "ipam": { + "type": "host-local", + "subnet": "usePodCidr" + }, + "policy": { + "type": "k8s", + "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__" + }, + "kubernetes": { + "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__", + "kubeconfig": "{{.KubeCfg}}" + } + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + } + ] } # Flannel network configuration. Mounted into the flannel container. @@ -197,6 +218,10 @@ spec: selector: matchLabels: k8s-app: canal + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 template: metadata: labels: @@ -207,21 +232,19 @@ spec: hostNetwork: true serviceAccountName: canal tolerations: - # this taint is set by all kubelets running '--cloud-provider=external' - # so we should tolerate it to schedule the canal pods - - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - effect: NoSchedule - # Allow the pod to run on the master abd etcd. This is required for - # the master to communicate with pods. + # Tolerate this effect so the pods will be schedulable at all times + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists - key: "node-role.kubernetes.io/master" operator: "Exists" - key: "node-role.kubernetes.io/etcd" operator: "Exists" effect: "NoExecute" - # Mark the pod as a critical add-on for rescheduling. - - key: "CriticalAddonsOnly" - operator: "Exists" # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 @@ -244,7 +267,7 @@ spec: # Cluster type to identify the deployment type - name: CLUSTER_TYPE value: "k8s,canal" - # Disable file logging so 'kubectl logs' works. + # Disable file logging so kubectl logs works. - name: CALICO_DISABLE_FILE_LOGGING value: "true" # Period, in seconds, at which felix re-applies all iptables state @@ -292,8 +315,9 @@ spec: - mountPath: /var/run/calico name: var-run-calico readOnly: false - - mountPath: /etc/kubernetes - name: etc-kubernetes + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni @@ -317,8 +341,6 @@ spec: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - mountPath: /etc/kubernetes - name: etc-kubernetes # This container runs flannel using the kube-subnet-mgr backend # for allocating subnets. - name: kube-flannel @@ -358,6 +380,9 @@ spec: - name: var-run-calico hostPath: path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico # Used to install CNI. - name: cni-bin-dir hostPath: @@ -372,44 +397,40 @@ spec: - name: flannel-cfg configMap: name: canal-config - - name: etc-kubernetes - hostPath: - path: /etc/kubernetes - # Create all the CustomResourceDefinitions needed for # Calico policy-only mode. --- apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico Global Felix Configuration +description: Calico Felix Configuration kind: CustomResourceDefinition metadata: - name: globalfelixconfigs.crd.projectcalico.org + name: felixconfigurations.crd.projectcalico.org spec: scope: Cluster group: crd.projectcalico.org version: v1 names: - kind: GlobalFelixConfig - plural: globalfelixconfigs - singular: globalfelixconfig + kind: FelixConfiguration + plural: felixconfigurations + singular: felixconfiguration --- apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico Global BGP Configuration +description: Calico BGP Configuration kind: CustomResourceDefinition metadata: - name: globalbgpconfigs.crd.projectcalico.org + name: bgpconfigurations.crd.projectcalico.org spec: scope: Cluster group: crd.projectcalico.org version: v1 names: - kind: GlobalBGPConfig - plural: globalbgpconfigs - singular: globalbgpconfig + kind: BGPConfiguration + plural: bgpconfigurations + singular: bgpconfiguration --- @@ -429,6 +450,22 @@ spec: --- +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico Cluster Information +kind: CustomResourceDefinition +metadata: + name: clusterinformations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: ClusterInformation + plural: clusterinformations + singular: clusterinformation + +--- + apiVersion: apiextensions.k8s.io/v1beta1 description: Calico Global Network Policies kind: CustomResourceDefinition @@ -445,8 +482,57 @@ spec: --- +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico Network Policies +kind: CustomResourceDefinition +metadata: + name: networkpolicies.crd.projectcalico.org +spec: + scope: Namespaced + group: crd.projectcalico.org + version: v1 + names: + kind: NetworkPolicy + plural: networkpolicies + singular: networkpolicy + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico Global Network Sets +kind: CustomResourceDefinition +metadata: + name: globalnetworksets.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkSet + plural: globalnetworksets + singular: globalnetworkset + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico Host Endpoints +kind: CustomResourceDefinition +metadata: + name: hostendpoints.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: HostEndpoint + plural: hostendpoints + singular: hostendpoint + +--- + apiVersion: v1 kind: ServiceAccount metadata: name: canal - namespace: kube-system` + namespace: kube-system +`