Extend rke to tolerate the Windows host

1. Support to configure Flannel as "host-gw" backend
2. Define the network component yaml and ingress
   controller yaml only schedule to non-Windows node
3. Support to configure Docker container's port publishing

cluster/defaults.go

@@ -190,6 +190,14 @@ func (c *Cluster) setClusterNetworkDefaults() {
 		networkPluginConfigDefaultsMap = map[string]string{
 			CalicoCloudProvider: DefaultNetworkCloudProvider,
 		}
+	case FlannelNetworkPlugin:
+		networkPluginConfigDefaultsMap = map[string]string{
+			FlannelBackendType: "vxlan",
+		}
+	case CanalNetworkPlugin:
+		networkPluginConfigDefaultsMap = map[string]string{
+			CanalFlannelBackendType: "vxlan",
+		}
 	}
 	if c.Network.CalicoNetworkProvider != nil {
 		setDefaultIfEmpty(&c.Network.CalicoNetworkProvider.CloudProvider, DefaultNetworkCloudProvider)
@@ -197,6 +205,7 @@ func (c *Cluster) setClusterNetworkDefaults() {
 	}
 	if c.Network.FlannelNetworkProvider != nil {
 		networkPluginConfigDefaultsMap[FlannelIface] = c.Network.FlannelNetworkProvider.Iface
+
 	}
 	if c.Network.CanalNetworkProvider != nil {
 		networkPluginConfigDefaultsMap[CanalIface] = c.Network.CanalNetworkProvider.Iface

cluster/network.go

@@ -40,12 +40,14 @@ const (
 
 	FlannelNetworkPlugin = "flannel"
 	FlannelIface         = "flannel_iface"
+	FlannelBackendType   = "flannel_backend_type"
 
 	CalicoNetworkPlugin = "calico"
 	CalicoCloudProvider = "calico_cloud_provider"
 
-	CanalNetworkPlugin = "canal"
-	CanalIface         = "canal_iface"
+	CanalNetworkPlugin      = "canal"
+	CanalIface              = "canal_iface"
+	CanalFlannelBackendType = "canal_flannel_backend_type"
 
 	WeaveNetworkPlugin = "weave"
 
@@ -83,6 +85,7 @@ const (
 	Calicoctl = "Calicoctl"
 
 	FlannelInterface = "FlannelInterface"
+	FlannelBackend   = "FlannelBackend"
 	CanalInterface   = "CanalInterface"
 	RBACConfig       = "RBACConfig"
 )
@@ -121,12 +124,15 @@ func (c *Cluster) deployNetworkPlugin(ctx context.Context) error {
 }
 
 func (c *Cluster) doFlannelDeploy(ctx context.Context) error {
-	flannelConfig := map[string]string{
+	flannelConfig := map[string]interface{}{
 		ClusterCIDR:      c.ClusterCIDR,
 		Image:            c.SystemImages.Flannel,
 		CNIImage:         c.SystemImages.FlannelCNI,
 		FlannelInterface: c.Network.Options[FlannelIface],
-		RBACConfig:       c.Authorization.Mode,
+		FlannelBackend: map[string]interface{}{
+			"Type": c.Network.Options[FlannelBackendType],
+		},
+		RBACConfig: c.Authorization.Mode,
 	}
 	pluginYaml, err := c.getNetworkPluginManifest(flannelConfig)
 	if err != nil {
@@ -137,7 +143,7 @@ func (c *Cluster) doFlannelDeploy(ctx context.Context) error {
 
 func (c *Cluster) doCalicoDeploy(ctx context.Context) error {
 	clientConfig := pki.GetConfigPath(pki.KubeNodeCertName)
-	calicoConfig := map[string]string{
+	calicoConfig := map[string]interface{}{
 		KubeCfg:       clientConfig,
 		ClusterCIDR:   c.ClusterCIDR,
 		CNIImage:      c.SystemImages.CalicoCNI,
@@ -155,7 +161,7 @@ func (c *Cluster) doCalicoDeploy(ctx context.Context) error {
 
 func (c *Cluster) doCanalDeploy(ctx context.Context) error {
 	clientConfig := pki.GetConfigPath(pki.KubeNodeCertName)
-	canalConfig := map[string]string{
+	canalConfig := map[string]interface{}{
 		ClientCertPath:  pki.GetCertPath(pki.KubeNodeCertName),
 		APIRoot:         "https://127.0.0.1:6443",
 		ClientKeyPath:   pki.GetKeyPath(pki.KubeNodeCertName),
@@ -167,6 +173,9 @@ func (c *Cluster) doCanalDeploy(ctx context.Context) error {
 		CanalFlannelImg: c.SystemImages.CanalFlannel,
 		RBACConfig:      c.Authorization.Mode,
 		CanalInterface:  c.Network.Options[CanalIface],
+		FlannelBackend: map[string]interface{}{
+			"Type": c.Network.Options[CanalFlannelBackendType],
+		},
 	}
 	pluginYaml, err := c.getNetworkPluginManifest(canalConfig)
 	if err != nil {
@@ -176,7 +185,7 @@ func (c *Cluster) doCanalDeploy(ctx context.Context) error {
 }
 
 func (c *Cluster) doWeaveDeploy(ctx context.Context) error {
-	weaveConfig := map[string]string{
+	weaveConfig := map[string]interface{}{
 		ClusterCIDR:        c.ClusterCIDR,
 		Image:              c.SystemImages.WeaveNode,
 		CNIImage:           c.SystemImages.WeaveCNI,
@@ -190,7 +199,7 @@ func (c *Cluster) doWeaveDeploy(ctx context.Context) error {
 	return c.doAddonDeploy(ctx, pluginYaml, NetworkPluginResourceName, true)
 }
 
-func (c *Cluster) getNetworkPluginManifest(pluginConfig map[string]string) (string, error) {
+func (c *Cluster) getNetworkPluginManifest(pluginConfig map[string]interface{}) (string, error) {
 	switch c.Network.Plugin {
 	case FlannelNetworkPlugin:
 		return templates.CompileTemplateFromMap(templates.FlannelTemplate, pluginConfig)

services/services.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/go-connections/nat"
 	"github.com/rancher/rke/docker"
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/log"
@@ -88,12 +89,14 @@ func GetProcessConfig(process v3.Process) (*container.Config, *container.HostCon
 	}
 	// var pidMode container.PidMode
 	// pidMode = process.PidMode
+	_, portBindings, _ := nat.ParsePortSpecs(process.Publish)
 	hostCfg := &container.HostConfig{
-		VolumesFrom: process.VolumesFrom,
-		Binds:       process.Binds,
-		NetworkMode: container.NetworkMode(process.NetworkMode),
-		PidMode:     container.PidMode(process.PidMode),
-		Privileged:  process.Privileged,
+		VolumesFrom:  process.VolumesFrom,
+		Binds:        process.Binds,
+		NetworkMode:  container.NetworkMode(process.NetworkMode),
+		PidMode:      container.PidMode(process.PidMode),
+		Privileged:   process.Privileged,
+		PortBindings: portBindings,
 	}
 	if len(process.RestartPolicy) > 0 {
 		hostCfg.RestartPolicy = container.RestartPolicy{Name: process.RestartPolicy}

templates/calico.go

@@ -173,6 +173,15 @@ spec:
         # if it ever gets evicted.
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                - key: beta.kubernetes.io/os
+                  operator: NotIn
+                  values:
+                    - windows
       hostNetwork: true
       tolerations:
         # Make sure calico/node gets scheduled on all nodes.

templates/canal.go

@@ -198,7 +198,7 @@ data:
     {
       "Network": "{{.ClusterCIDR}}",
       "Backend": {
-        "Type": "vxlan"
+        "Type": "{{.FlannelBackend.Type}}"
       }
     }
 
@@ -229,6 +229,15 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                - key: beta.kubernetes.io/os
+                  operator: NotIn
+                  values:
+                    - windows
       hostNetwork: true
       serviceAccountName: canal
       tolerations:

templates/flannel.go

@@ -75,7 +75,7 @@ data:
     {
       "Network": "{{.ClusterCIDR}}",
       "Backend": {
-        "Type": "vxlan"
+        "Type": "{{.FlannelBackend.Type}}"
       }
     }
 ---
@@ -94,6 +94,15 @@ spec:
         tier: node
         k8s-app: flannel
     spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                - key: beta.kubernetes.io/os
+                  operator: NotIn
+                  values:
+                    - windows
       serviceAccountName: flannel
       containers:
       - name: kube-flannel

templates/nginx-ingress.go

@@ -176,6 +176,15 @@ spec:
         prometheus.io/port: '10254'
         prometheus.io/scrape: 'true'
     spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                - key: beta.kubernetes.io/os
+                  operator: NotIn
+                  values:
+                    - windows
       hostNetwork: true
       nodeSelector:
       {{ range $k, $v := .NodeSelector }}

templates/weave.go

@@ -24,6 +24,15 @@ items:
           labels:
             name: weave-net
         spec:
+          affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                  - matchExpressions:
+                    - key: beta.kubernetes.io/os
+                      operator: NotIn
+                      values:
+                        - windows
           containers:
             - name: weave
               command: