Add dind mode to rke

cmd/remove.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/rancher/rke/cluster"
+	"github.com/rancher/rke/dind"
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/k8s"
 	"github.com/rancher/rke/log"
@@ -31,7 +32,11 @@ func RemoveCommand() cli.Command {
 		},
 		cli.BoolFlag{
 			Name:  "local",
-			Usage: "Deploy Kubernetes cluster locally",
+			Usage: "Remove Kubernetes cluster locally",
+		},
+		cli.BoolFlag{
+			Name:  "dind",
+			Usage: "Remove Kubernetes cluster deployed in dind mode",
 		},
 	}
 
@@ -94,6 +99,9 @@ func clusterRemoveFromCli(ctx *cli.Context) error {
 	if ctx.Bool("local") {
 		return clusterRemoveLocal(ctx)
 	}
+	if ctx.Bool("dind") {
+		return clusterRemoveDind(ctx)
+	}
 	clusterFilePath = filePath
 	rkeConfig, err := cluster.ParseConfig(clusterFile)
 	if err != nil {
@@ -130,3 +138,30 @@ func clusterRemoveLocal(ctx *cli.Context) error {
 
 	return ClusterRemove(context.Background(), rkeConfig, nil, nil, true, "")
 }
+
+func clusterRemoveDind(ctx *cli.Context) error {
+	clusterFile, filePath, err := resolveClusterFile(ctx)
+	if err != nil {
+		return fmt.Errorf("Failed to resolve cluster file: %v", err)
+	}
+
+	rkeConfig, err := cluster.ParseConfig(clusterFile)
+	if err != nil {
+		return fmt.Errorf("Failed to parse cluster file: %v", err)
+	}
+
+	rkeConfig, err = setOptionsFromCLI(ctx, rkeConfig)
+	if err != nil {
+		return err
+	}
+
+	for _, node := range rkeConfig.Nodes {
+		if err = dind.RmoveDindContainer(context.Background(), node.Address); err != nil {
+			return err
+		}
+	}
+	localKubeConfigPath := pki.GetLocalKubeConfig(filePath, "")
+	// remove the kube config file
+	pki.RemoveAdminConfig(context.Background(), localKubeConfigPath)
+	return err
+}

cmd/up.go

@@ -4,19 +4,24 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/rancher/rke/cluster"
+	"github.com/rancher/rke/dind"
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/k8s"
 	"github.com/rancher/rke/log"
 	"github.com/rancher/rke/pki"
 	"github.com/rancher/types/apis/management.cattle.io/v3"
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 	"k8s.io/client-go/util/cert"
 )
 
 var clusterFilePath string
 
+const DINDWaitTime = 3
+
 func UpCommand() cli.Command {
 	upFlags := []cli.Flag{
 		cli.StringFlag{
@@ -29,6 +34,14 @@ func UpCommand() cli.Command {
 			Name:  "local",
 			Usage: "Deploy Kubernetes cluster locally",
 		},
+		cli.BoolFlag{
+			Name:  "dind",
+			Usage: "Deploy Kubernetes cluster in docker containers (experimental)",
+		},
+		cli.StringFlag{
+			Name:  "dind-subnet",
+			Usage: "User defined network to deploy k8s within (experimental)",
+		},
 		cli.BoolFlag{
 			Name:  "update-only",
 			Usage: "Skip idempotent deployment of control and etcd plane",
@@ -163,6 +176,9 @@ func clusterUpFromCli(ctx *cli.Context) error {
 	if ctx.Bool("local") {
 		return clusterUpLocal(ctx)
 	}
+	if ctx.Bool("dind") {
+		return clusterUpDind(ctx)
+	}
 	clusterFile, filePath, err := resolveClusterFile(ctx)
 	if err != nil {
 		return fmt.Errorf("Failed to resolve cluster file: %v", err)
@@ -205,3 +221,63 @@ func clusterUpLocal(ctx *cli.Context) error {
 	_, _, _, _, _, err = ClusterUp(context.Background(), rkeConfig, nil, hosts.LocalHealthcheckFactory, nil, true, "", false, false)
 	return err
 }
+
+func clusterUpDind(ctx *cli.Context) error {
+	// get dind config
+	rkeConfig, disablePortCheck, dindSubnet, err := getDindConfig(ctx)
+	if err != nil {
+		return err
+	}
+	// setup dind environment
+	if err = createDINDEnv(context.Background(), dindSubnet, rkeConfig); err != nil {
+		return err
+	}
+	// start cluster
+	_, _, _, _, _, err = ClusterUp(context.Background(), rkeConfig, hosts.DindConnFactory, hosts.DindHealthcheckConnFactory, nil, false, "", false, disablePortCheck)
+	return err
+}
+
+func getDindConfig(ctx *cli.Context) (*v3.RancherKubernetesEngineConfig, bool, string, error) {
+	disablePortCheck := ctx.Bool("disable-port-check")
+	dindSubnet := ctx.String("dind-subnet")
+	clusterFile, filePath, err := resolveClusterFile(ctx)
+	if err != nil {
+		return nil, disablePortCheck, dindSubnet, fmt.Errorf("Failed to resolve cluster file: %v", err)
+	}
+	clusterFilePath = filePath
+
+	rkeConfig, err := cluster.ParseConfig(clusterFile)
+	if err != nil {
+		return nil, disablePortCheck, dindSubnet, fmt.Errorf("Failed to parse cluster file: %v", err)
+	}
+
+	rkeConfig, err = setOptionsFromCLI(ctx, rkeConfig)
+	if err != nil {
+		return nil, disablePortCheck, dindSubnet, err
+	}
+	// Setting conntrack max for kubeproxy to 0
+	if rkeConfig.Services.Kubeproxy.ExtraArgs == nil {
+		rkeConfig.Services.Kubeproxy.ExtraArgs = make(map[string]string)
+	}
+	rkeConfig.Services.Kubeproxy.ExtraArgs["conntrack-max-per-core"] = "0"
+
+	return rkeConfig, disablePortCheck, dindSubnet, nil
+}
+
+func createDINDEnv(ctx context.Context, dindSubnet string, rkeConfig *v3.RancherKubernetesEngineConfig) error {
+	if dindSubnet == "" {
+		logrus.Infof("[%s] dind subnet didn't get specified, using default subnet [%s]", dind.DINDPlane, dind.DINDSubnet)
+		dindSubnet = dind.DINDSubnet
+	}
+	if err := dind.CreateDindNetwork(ctx, dindSubnet); err != nil {
+		return fmt.Errorf("Failed to create dind network: %v", err)
+	}
+
+	for _, node := range rkeConfig.Nodes {
+		if err := dind.StartUpDindContainer(ctx, node.Address, dind.DINDNetwork); err != nil {
+			return err
+		}
+	}
+	time.Sleep(DINDWaitTime * time.Second)
+	return nil
+}

dind/dind.go

@@ -0,0 +1,146 @@
+package dind
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/client"
+	"github.com/rancher/rke/docker"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	DINDImage           = "docker:17.03-dind"
+	DINDContainerPrefix = "rke-dind-"
+	DINDPlane           = "dind"
+	DINDNetwork         = "dind-network"
+	DINDSubnet          = "172.18.0.0/16"
+)
+
+func StartUpDindContainer(ctx context.Context, dindAddress, dindNetwork string) error {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		return err
+	}
+	// its recommended to use host's storage driver
+	dockerInfo, err := cli.Info(ctx)
+	if err != nil {
+		return err
+	}
+	storageDriver := dockerInfo.Driver
+	// Get dind container name
+	containerName := DINDContainerPrefix + dindAddress
+	_, err = cli.ContainerInspect(ctx, containerName)
+	if err != nil {
+		if !client.IsErrNotFound(err) {
+			return err
+		}
+		if err := docker.UseLocalOrPull(ctx, cli, cli.DaemonHost(), DINDImage, DINDPlane, nil); err != nil {
+			return err
+		}
+		binds := []string{
+			fmt.Sprintf("/var/lib/kubelet-%s:/var/lib/kubelet:shared", containerName),
+			"/etc/resolv.conf:/etc/resolv.conf",
+		}
+		imageCfg := &container.Config{
+			Image: DINDImage,
+			Entrypoint: []string{
+				"sh",
+				"-c",
+				"mount --make-shared / && " +
+					"mount --make-shared /var/lib/docker && " +
+					"dockerd-entrypoint.sh --storage-driver=" + storageDriver,
+			},
+		}
+		hostCfg := &container.HostConfig{
+			Privileged: true,
+			Binds:      binds,
+		}
+		netCfg := &network.NetworkingConfig{
+			EndpointsConfig: map[string]*network.EndpointSettings{
+				dindNetwork: &network.EndpointSettings{
+					IPAMConfig: &network.EndpointIPAMConfig{
+						IPv4Address: dindAddress,
+					},
+				},
+			},
+		}
+		resp, err := cli.ContainerCreate(ctx, imageCfg, hostCfg, netCfg, containerName)
+		if err != nil {
+			return fmt.Errorf("Failed to create [%s] container on host [%s]: %v", containerName, cli.DaemonHost(), err)
+		}
+
+		if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
+			return fmt.Errorf("Failed to start [%s] container on host [%s]: %v", containerName, cli.DaemonHost(), err)
+		}
+		logrus.Infof("[%s] Successfully started [%s] container on host [%s]", DINDPlane, containerName, cli.DaemonHost())
+		return nil
+	}
+	logrus.Infof("[%s] container [%s] is already running on host[%s]", DINDPlane, containerName, cli.DaemonHost())
+	return nil
+}
+
+func RmoveDindContainer(ctx context.Context, dindAddress string) error {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		return err
+	}
+	containerName := DINDContainerPrefix + dindAddress
+	logrus.Infof("[%s] Removing dind container [%s] on host [%s]", DINDPlane, containerName, cli.DaemonHost())
+	_, err = cli.ContainerInspect(ctx, containerName)
+	if err != nil {
+		if !client.IsErrNotFound(err) {
+			return nil
+		}
+	}
+	if err := cli.ContainerRemove(ctx, containerName, types.ContainerRemoveOptions{
+		Force:         true,
+		RemoveVolumes: true}); err != nil {
+		return fmt.Errorf("Failed to remove dind container [%s] on host [%s]: %v", containerName, cli.DaemonHost(), err)
+	}
+	logrus.Infof("[%s] Successfully Removed dind container [%s] on host [%s]", DINDPlane, containerName, cli.DaemonHost())
+	return nil
+}
+
+func CreateDindNetwork(ctx context.Context, dindSubnet string) error {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		return err
+	}
+	networkList, err := cli.NetworkList(ctx, types.NetworkListOptions{})
+	for _, net := range networkList {
+		if DINDNetwork == net.Name {
+			subnetFound := false
+			for _, netConfig := range net.IPAM.Config {
+				if netConfig.Subnet == dindSubnet {
+					subnetFound = true
+					break
+				}
+			}
+			if !subnetFound {
+				return fmt.Errorf("dind network [%s] exist but has different subnet than specified", DINDNetwork)
+			}
+			logrus.Infof("[%s] dind network [%s] with subnet [%s] already created", DINDPlane, DINDNetwork, dindSubnet)
+			return nil
+		}
+	}
+	logrus.Infof("[%s] creating dind network [%s] with subnet [%s]", DINDPlane, DINDNetwork, dindSubnet)
+	_, err = cli.NetworkCreate(ctx, DINDNetwork, types.NetworkCreate{
+		Driver: "bridge",
+		IPAM: &network.IPAM{
+			Config: []network.IPAMConfig{
+				network.IPAMConfig{
+					Subnet: dindSubnet,
+				},
+			},
+		},
+	})
+	if err != nil {
+		return err
+	}
+	logrus.Infof("[%s] Successfully Created dind network [%s] with subnet [%s]", DINDPlane, DINDNetwork, dindSubnet)
+	return nil
+}

hosts/dialer.go

@@ -12,7 +12,7 @@ import (
 )
 
 const (
-	DockerDialerTimeout = 30
+	DockerDialerTimeout = 50
 )
 
 type DialerFactory func(h *Host) (func(network, address string) (net.Conn, error), error)

hosts/dind.go

@@ -0,0 +1,41 @@
+package hosts
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+)
+
+const (
+	DINDPort = "2375"
+)
+
+type dindDialer struct {
+	Address string
+	Port    string
+	Network string
+}
+
+func DindConnFactory(h *Host) (func(network, address string) (net.Conn, error), error) {
+	newDindDialer := &dindDialer{
+		Address: h.Address,
+		Port:    DINDPort,
+	}
+	return newDindDialer.Dial, nil
+}
+
+func DindHealthcheckConnFactory(h *Host) (func(network, address string) (net.Conn, error), error) {
+	newDindDialer := &dindDialer{
+		Address: h.Address,
+		Port:    strconv.Itoa(h.LocalConnPort),
+	}
+	return newDindDialer.Dial, nil
+}
+
+func (d *dindDialer) Dial(network, addr string) (net.Conn, error) {
+	conn, err := net.Dial(network, d.Address+":"+d.Port)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to dial dind address [%s]: %v", addr, err)
+	}
+	return conn, err
+}

services/healthcheck.go

@@ -56,7 +56,7 @@ func runHealthcheck(ctx context.Context, host *hosts.Host, serviceName string, l
 	}
 	client, err := getHealthCheckHTTPClient(host, port, localConnDialerFactory, &x509Pair)
 	if err != nil {
-		return fmt.Errorf("Failed to initiate new HTTP client for service [%s] for host [%s]", serviceName, host.Address)
+		return fmt.Errorf("Failed to initiate new HTTP client for service [%s] for host [%s]: %v", serviceName, host.Address, err)
 	}
 	for retries := 0; retries < 10; retries++ {
 		if err = getHealthz(client, serviceName, host.Address, url); err != nil {