From 247f4c9450aafdb6dfb754d48a0a2f98f6c2249e Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Tue, 10 Jul 2018 21:21:27 +0200 Subject: [PATCH] Add dind mode to rke --- cmd/remove.go | 37 +++++++++- cmd/up.go | 76 +++++++++++++++++++++ dind/dind.go | 146 ++++++++++++++++++++++++++++++++++++++++ hosts/dialer.go | 2 +- hosts/dind.go | 41 +++++++++++ services/healthcheck.go | 2 +- 6 files changed, 301 insertions(+), 3 deletions(-) create mode 100644 dind/dind.go create mode 100644 hosts/dind.go diff --git a/cmd/remove.go b/cmd/remove.go index 2ab7ff59..59e1cc94 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/rancher/rke/cluster" + "github.com/rancher/rke/dind" "github.com/rancher/rke/hosts" "github.com/rancher/rke/k8s" "github.com/rancher/rke/log" @@ -31,7 +32,11 @@ func RemoveCommand() cli.Command { }, cli.BoolFlag{ Name: "local", - Usage: "Deploy Kubernetes cluster locally", + Usage: "Remove Kubernetes cluster locally", + }, + cli.BoolFlag{ + Name: "dind", + Usage: "Remove Kubernetes cluster deployed in dind mode", }, } @@ -94,6 +99,9 @@ func clusterRemoveFromCli(ctx *cli.Context) error { if ctx.Bool("local") { return clusterRemoveLocal(ctx) } + if ctx.Bool("dind") { + return clusterRemoveDind(ctx) + } clusterFilePath = filePath rkeConfig, err := cluster.ParseConfig(clusterFile) if err != nil { @@ -130,3 +138,30 @@ func clusterRemoveLocal(ctx *cli.Context) error { return ClusterRemove(context.Background(), rkeConfig, nil, nil, true, "") } + +func clusterRemoveDind(ctx *cli.Context) error { + clusterFile, filePath, err := resolveClusterFile(ctx) + if err != nil { + return fmt.Errorf("Failed to resolve cluster file: %v", err) + } + + rkeConfig, err := cluster.ParseConfig(clusterFile) + if err != nil { + return fmt.Errorf("Failed to parse cluster file: %v", err) + } + + rkeConfig, err = setOptionsFromCLI(ctx, rkeConfig) + if err != nil { + return err + } + + for _, node := range rkeConfig.Nodes { + if err = dind.RmoveDindContainer(context.Background(), node.Address); err != nil { + return err + } + } + localKubeConfigPath := pki.GetLocalKubeConfig(filePath, "") + // remove the kube config file + pki.RemoveAdminConfig(context.Background(), localKubeConfigPath) + return err +} diff --git a/cmd/up.go b/cmd/up.go index 45025836..3a0aa12c 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -4,19 +4,24 @@ import ( "context" "fmt" "strings" + "time" "github.com/rancher/rke/cluster" + "github.com/rancher/rke/dind" "github.com/rancher/rke/hosts" "github.com/rancher/rke/k8s" "github.com/rancher/rke/log" "github.com/rancher/rke/pki" "github.com/rancher/types/apis/management.cattle.io/v3" + "github.com/sirupsen/logrus" "github.com/urfave/cli" "k8s.io/client-go/util/cert" ) var clusterFilePath string +const DINDWaitTime = 3 + func UpCommand() cli.Command { upFlags := []cli.Flag{ cli.StringFlag{ @@ -29,6 +34,14 @@ func UpCommand() cli.Command { Name: "local", Usage: "Deploy Kubernetes cluster locally", }, + cli.BoolFlag{ + Name: "dind", + Usage: "Deploy Kubernetes cluster in docker containers (experimental)", + }, + cli.StringFlag{ + Name: "dind-subnet", + Usage: "User defined network to deploy k8s within (experimental)", + }, cli.BoolFlag{ Name: "update-only", Usage: "Skip idempotent deployment of control and etcd plane", @@ -163,6 +176,9 @@ func clusterUpFromCli(ctx *cli.Context) error { if ctx.Bool("local") { return clusterUpLocal(ctx) } + if ctx.Bool("dind") { + return clusterUpDind(ctx) + } clusterFile, filePath, err := resolveClusterFile(ctx) if err != nil { return fmt.Errorf("Failed to resolve cluster file: %v", err) @@ -205,3 +221,63 @@ func clusterUpLocal(ctx *cli.Context) error { _, _, _, _, _, err = ClusterUp(context.Background(), rkeConfig, nil, hosts.LocalHealthcheckFactory, nil, true, "", false, false) return err } + +func clusterUpDind(ctx *cli.Context) error { + // get dind config + rkeConfig, disablePortCheck, dindSubnet, err := getDindConfig(ctx) + if err != nil { + return err + } + // setup dind environment + if err = createDINDEnv(context.Background(), dindSubnet, rkeConfig); err != nil { + return err + } + // start cluster + _, _, _, _, _, err = ClusterUp(context.Background(), rkeConfig, hosts.DindConnFactory, hosts.DindHealthcheckConnFactory, nil, false, "", false, disablePortCheck) + return err +} + +func getDindConfig(ctx *cli.Context) (*v3.RancherKubernetesEngineConfig, bool, string, error) { + disablePortCheck := ctx.Bool("disable-port-check") + dindSubnet := ctx.String("dind-subnet") + clusterFile, filePath, err := resolveClusterFile(ctx) + if err != nil { + return nil, disablePortCheck, dindSubnet, fmt.Errorf("Failed to resolve cluster file: %v", err) + } + clusterFilePath = filePath + + rkeConfig, err := cluster.ParseConfig(clusterFile) + if err != nil { + return nil, disablePortCheck, dindSubnet, fmt.Errorf("Failed to parse cluster file: %v", err) + } + + rkeConfig, err = setOptionsFromCLI(ctx, rkeConfig) + if err != nil { + return nil, disablePortCheck, dindSubnet, err + } + // Setting conntrack max for kubeproxy to 0 + if rkeConfig.Services.Kubeproxy.ExtraArgs == nil { + rkeConfig.Services.Kubeproxy.ExtraArgs = make(map[string]string) + } + rkeConfig.Services.Kubeproxy.ExtraArgs["conntrack-max-per-core"] = "0" + + return rkeConfig, disablePortCheck, dindSubnet, nil +} + +func createDINDEnv(ctx context.Context, dindSubnet string, rkeConfig *v3.RancherKubernetesEngineConfig) error { + if dindSubnet == "" { + logrus.Infof("[%s] dind subnet didn't get specified, using default subnet [%s]", dind.DINDPlane, dind.DINDSubnet) + dindSubnet = dind.DINDSubnet + } + if err := dind.CreateDindNetwork(ctx, dindSubnet); err != nil { + return fmt.Errorf("Failed to create dind network: %v", err) + } + + for _, node := range rkeConfig.Nodes { + if err := dind.StartUpDindContainer(ctx, node.Address, dind.DINDNetwork); err != nil { + return err + } + } + time.Sleep(DINDWaitTime * time.Second) + return nil +} diff --git a/dind/dind.go b/dind/dind.go new file mode 100644 index 00000000..c47bebcf --- /dev/null +++ b/dind/dind.go @@ -0,0 +1,146 @@ +package dind + +import ( + "context" + "fmt" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" + "github.com/docker/docker/client" + "github.com/rancher/rke/docker" + "github.com/sirupsen/logrus" +) + +const ( + DINDImage = "docker:17.03-dind" + DINDContainerPrefix = "rke-dind-" + DINDPlane = "dind" + DINDNetwork = "dind-network" + DINDSubnet = "172.18.0.0/16" +) + +func StartUpDindContainer(ctx context.Context, dindAddress, dindNetwork string) error { + cli, err := client.NewEnvClient() + if err != nil { + return err + } + // its recommended to use host's storage driver + dockerInfo, err := cli.Info(ctx) + if err != nil { + return err + } + storageDriver := dockerInfo.Driver + // Get dind container name + containerName := DINDContainerPrefix + dindAddress + _, err = cli.ContainerInspect(ctx, containerName) + if err != nil { + if !client.IsErrNotFound(err) { + return err + } + if err := docker.UseLocalOrPull(ctx, cli, cli.DaemonHost(), DINDImage, DINDPlane, nil); err != nil { + return err + } + binds := []string{ + fmt.Sprintf("/var/lib/kubelet-%s:/var/lib/kubelet:shared", containerName), + "/etc/resolv.conf:/etc/resolv.conf", + } + imageCfg := &container.Config{ + Image: DINDImage, + Entrypoint: []string{ + "sh", + "-c", + "mount --make-shared / && " + + "mount --make-shared /var/lib/docker && " + + "dockerd-entrypoint.sh --storage-driver=" + storageDriver, + }, + } + hostCfg := &container.HostConfig{ + Privileged: true, + Binds: binds, + } + netCfg := &network.NetworkingConfig{ + EndpointsConfig: map[string]*network.EndpointSettings{ + dindNetwork: &network.EndpointSettings{ + IPAMConfig: &network.EndpointIPAMConfig{ + IPv4Address: dindAddress, + }, + }, + }, + } + resp, err := cli.ContainerCreate(ctx, imageCfg, hostCfg, netCfg, containerName) + if err != nil { + return fmt.Errorf("Failed to create [%s] container on host [%s]: %v", containerName, cli.DaemonHost(), err) + } + + if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { + return fmt.Errorf("Failed to start [%s] container on host [%s]: %v", containerName, cli.DaemonHost(), err) + } + logrus.Infof("[%s] Successfully started [%s] container on host [%s]", DINDPlane, containerName, cli.DaemonHost()) + return nil + } + logrus.Infof("[%s] container [%s] is already running on host[%s]", DINDPlane, containerName, cli.DaemonHost()) + return nil +} + +func RmoveDindContainer(ctx context.Context, dindAddress string) error { + cli, err := client.NewEnvClient() + if err != nil { + return err + } + containerName := DINDContainerPrefix + dindAddress + logrus.Infof("[%s] Removing dind container [%s] on host [%s]", DINDPlane, containerName, cli.DaemonHost()) + _, err = cli.ContainerInspect(ctx, containerName) + if err != nil { + if !client.IsErrNotFound(err) { + return nil + } + } + if err := cli.ContainerRemove(ctx, containerName, types.ContainerRemoveOptions{ + Force: true, + RemoveVolumes: true}); err != nil { + return fmt.Errorf("Failed to remove dind container [%s] on host [%s]: %v", containerName, cli.DaemonHost(), err) + } + logrus.Infof("[%s] Successfully Removed dind container [%s] on host [%s]", DINDPlane, containerName, cli.DaemonHost()) + return nil +} + +func CreateDindNetwork(ctx context.Context, dindSubnet string) error { + cli, err := client.NewEnvClient() + if err != nil { + return err + } + networkList, err := cli.NetworkList(ctx, types.NetworkListOptions{}) + for _, net := range networkList { + if DINDNetwork == net.Name { + subnetFound := false + for _, netConfig := range net.IPAM.Config { + if netConfig.Subnet == dindSubnet { + subnetFound = true + break + } + } + if !subnetFound { + return fmt.Errorf("dind network [%s] exist but has different subnet than specified", DINDNetwork) + } + logrus.Infof("[%s] dind network [%s] with subnet [%s] already created", DINDPlane, DINDNetwork, dindSubnet) + return nil + } + } + logrus.Infof("[%s] creating dind network [%s] with subnet [%s]", DINDPlane, DINDNetwork, dindSubnet) + _, err = cli.NetworkCreate(ctx, DINDNetwork, types.NetworkCreate{ + Driver: "bridge", + IPAM: &network.IPAM{ + Config: []network.IPAMConfig{ + network.IPAMConfig{ + Subnet: dindSubnet, + }, + }, + }, + }) + if err != nil { + return err + } + logrus.Infof("[%s] Successfully Created dind network [%s] with subnet [%s]", DINDPlane, DINDNetwork, dindSubnet) + return nil +} diff --git a/hosts/dialer.go b/hosts/dialer.go index 74132bfd..5b1e6ca5 100644 --- a/hosts/dialer.go +++ b/hosts/dialer.go @@ -12,7 +12,7 @@ import ( ) const ( - DockerDialerTimeout = 30 + DockerDialerTimeout = 50 ) type DialerFactory func(h *Host) (func(network, address string) (net.Conn, error), error) diff --git a/hosts/dind.go b/hosts/dind.go new file mode 100644 index 00000000..7e6a09df --- /dev/null +++ b/hosts/dind.go @@ -0,0 +1,41 @@ +package hosts + +import ( + "fmt" + "net" + "strconv" +) + +const ( + DINDPort = "2375" +) + +type dindDialer struct { + Address string + Port string + Network string +} + +func DindConnFactory(h *Host) (func(network, address string) (net.Conn, error), error) { + newDindDialer := &dindDialer{ + Address: h.Address, + Port: DINDPort, + } + return newDindDialer.Dial, nil +} + +func DindHealthcheckConnFactory(h *Host) (func(network, address string) (net.Conn, error), error) { + newDindDialer := &dindDialer{ + Address: h.Address, + Port: strconv.Itoa(h.LocalConnPort), + } + return newDindDialer.Dial, nil +} + +func (d *dindDialer) Dial(network, addr string) (net.Conn, error) { + conn, err := net.Dial(network, d.Address+":"+d.Port) + if err != nil { + return nil, fmt.Errorf("Failed to dial dind address [%s]: %v", addr, err) + } + return conn, err +} diff --git a/services/healthcheck.go b/services/healthcheck.go index 3ed25328..6287305c 100644 --- a/services/healthcheck.go +++ b/services/healthcheck.go @@ -56,7 +56,7 @@ func runHealthcheck(ctx context.Context, host *hosts.Host, serviceName string, l } client, err := getHealthCheckHTTPClient(host, port, localConnDialerFactory, &x509Pair) if err != nil { - return fmt.Errorf("Failed to initiate new HTTP client for service [%s] for host [%s]", serviceName, host.Address) + return fmt.Errorf("Failed to initiate new HTTP client for service [%s] for host [%s]: %v", serviceName, host.Address, err) } for retries := 0; retries < 10; retries++ { if err = getHealthz(client, serviceName, host.Address, url); err != nil {