diff --git a/cmd/up.go b/cmd/up.go index 3a0aa12c..a56584d5 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -13,7 +13,6 @@ import ( "github.com/rancher/rke/log" "github.com/rancher/rke/pki" "github.com/rancher/types/apis/management.cattle.io/v3" - "github.com/sirupsen/logrus" "github.com/urfave/cli" "k8s.io/client-go/util/cert" ) @@ -38,10 +37,6 @@ func UpCommand() cli.Command { Name: "dind", Usage: "Deploy Kubernetes cluster in docker containers (experimental)", }, - cli.StringFlag{ - Name: "dind-subnet", - Usage: "User defined network to deploy k8s within (experimental)", - }, cli.BoolFlag{ Name: "update-only", Usage: "Skip idempotent deployment of control and etcd plane", @@ -224,12 +219,12 @@ func clusterUpLocal(ctx *cli.Context) error { func clusterUpDind(ctx *cli.Context) error { // get dind config - rkeConfig, disablePortCheck, dindSubnet, err := getDindConfig(ctx) + rkeConfig, disablePortCheck, err := getDindConfig(ctx) if err != nil { return err } // setup dind environment - if err = createDINDEnv(context.Background(), dindSubnet, rkeConfig); err != nil { + if err = createDINDEnv(context.Background(), rkeConfig); err != nil { return err } // start cluster @@ -237,23 +232,22 @@ func clusterUpDind(ctx *cli.Context) error { return err } -func getDindConfig(ctx *cli.Context) (*v3.RancherKubernetesEngineConfig, bool, string, error) { +func getDindConfig(ctx *cli.Context) (*v3.RancherKubernetesEngineConfig, bool, error) { disablePortCheck := ctx.Bool("disable-port-check") - dindSubnet := ctx.String("dind-subnet") clusterFile, filePath, err := resolveClusterFile(ctx) if err != nil { - return nil, disablePortCheck, dindSubnet, fmt.Errorf("Failed to resolve cluster file: %v", err) + return nil, disablePortCheck, fmt.Errorf("Failed to resolve cluster file: %v", err) } clusterFilePath = filePath rkeConfig, err := cluster.ParseConfig(clusterFile) if err != nil { - return nil, disablePortCheck, dindSubnet, fmt.Errorf("Failed to parse cluster file: %v", err) + return nil, disablePortCheck, fmt.Errorf("Failed to parse cluster file: %v", err) } rkeConfig, err = setOptionsFromCLI(ctx, rkeConfig) if err != nil { - return nil, disablePortCheck, dindSubnet, err + return nil, disablePortCheck, err } // Setting conntrack max for kubeproxy to 0 if rkeConfig.Services.Kubeproxy.ExtraArgs == nil { @@ -261,22 +255,19 @@ func getDindConfig(ctx *cli.Context) (*v3.RancherKubernetesEngineConfig, bool, s } rkeConfig.Services.Kubeproxy.ExtraArgs["conntrack-max-per-core"] = "0" - return rkeConfig, disablePortCheck, dindSubnet, nil + return rkeConfig, disablePortCheck, nil } -func createDINDEnv(ctx context.Context, dindSubnet string, rkeConfig *v3.RancherKubernetesEngineConfig) error { - if dindSubnet == "" { - logrus.Infof("[%s] dind subnet didn't get specified, using default subnet [%s]", dind.DINDPlane, dind.DINDSubnet) - dindSubnet = dind.DINDSubnet - } - if err := dind.CreateDindNetwork(ctx, dindSubnet); err != nil { - return fmt.Errorf("Failed to create dind network: %v", err) - } - - for _, node := range rkeConfig.Nodes { - if err := dind.StartUpDindContainer(ctx, node.Address, dind.DINDNetwork); err != nil { +func createDINDEnv(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig) error { + for i := range rkeConfig.Nodes { + address, err := dind.StartUpDindContainer(ctx, rkeConfig.Nodes[i].Address, dind.DINDNetwork) + if err != nil { return err } + if rkeConfig.Nodes[i].HostnameOverride == "" { + rkeConfig.Nodes[i].HostnameOverride = rkeConfig.Nodes[i].Address + } + rkeConfig.Nodes[i].Address = address } time.Sleep(DINDWaitTime * time.Second) return nil diff --git a/dind/dind.go b/dind/dind.go index c47bebcf..302c2013 100644 --- a/dind/dind.go +++ b/dind/dind.go @@ -6,7 +6,6 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/rancher/rke/docker" "github.com/sirupsen/logrus" @@ -14,32 +13,32 @@ import ( const ( DINDImage = "docker:17.03-dind" - DINDContainerPrefix = "rke-dind-" + DINDContainerPrefix = "rke-dind" DINDPlane = "dind" DINDNetwork = "dind-network" DINDSubnet = "172.18.0.0/16" ) -func StartUpDindContainer(ctx context.Context, dindAddress, dindNetwork string) error { +func StartUpDindContainer(ctx context.Context, dindAddress, dindNetwork string) (string, error) { cli, err := client.NewEnvClient() if err != nil { - return err + return "", err } // its recommended to use host's storage driver dockerInfo, err := cli.Info(ctx) if err != nil { - return err + return "", err } storageDriver := dockerInfo.Driver // Get dind container name - containerName := DINDContainerPrefix + dindAddress + containerName := fmt.Sprintf("%s-%s", DINDContainerPrefix, dindAddress) _, err = cli.ContainerInspect(ctx, containerName) if err != nil { if !client.IsErrNotFound(err) { - return err + return "", err } if err := docker.UseLocalOrPull(ctx, cli, cli.DaemonHost(), DINDImage, DINDPlane, nil); err != nil { - return err + return "", err } binds := []string{ fmt.Sprintf("/var/lib/kubelet-%s:/var/lib/kubelet:shared", containerName), @@ -54,33 +53,36 @@ func StartUpDindContainer(ctx context.Context, dindAddress, dindNetwork string) "mount --make-shared /var/lib/docker && " + "dockerd-entrypoint.sh --storage-driver=" + storageDriver, }, + Hostname: dindAddress, } hostCfg := &container.HostConfig{ Privileged: true, Binds: binds, } - netCfg := &network.NetworkingConfig{ - EndpointsConfig: map[string]*network.EndpointSettings{ - dindNetwork: &network.EndpointSettings{ - IPAMConfig: &network.EndpointIPAMConfig{ - IPv4Address: dindAddress, - }, - }, - }, - } - resp, err := cli.ContainerCreate(ctx, imageCfg, hostCfg, netCfg, containerName) + resp, err := cli.ContainerCreate(ctx, imageCfg, hostCfg, nil, containerName) if err != nil { - return fmt.Errorf("Failed to create [%s] container on host [%s]: %v", containerName, cli.DaemonHost(), err) + return "", fmt.Errorf("Failed to create [%s] container on host [%s]: %v", containerName, cli.DaemonHost(), err) } if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { - return fmt.Errorf("Failed to start [%s] container on host [%s]: %v", containerName, cli.DaemonHost(), err) + return "", fmt.Errorf("Failed to start [%s] container on host [%s]: %v", containerName, cli.DaemonHost(), err) } logrus.Infof("[%s] Successfully started [%s] container on host [%s]", DINDPlane, containerName, cli.DaemonHost()) - return nil + dindContainer, err := cli.ContainerInspect(ctx, containerName) + if err != nil { + return "", fmt.Errorf("Failed to get the address of container [%s] on host [%s]: %v", containerName, cli.DaemonHost(), err) + } + dindIPAddress := dindContainer.NetworkSettings.IPAddress + + return dindIPAddress, nil } + dindContainer, err := cli.ContainerInspect(ctx, containerName) + if err != nil { + return "", fmt.Errorf("Failed to get the address of container [%s] on host [%s]: %v", containerName, cli.DaemonHost(), err) + } + dindIPAddress := dindContainer.NetworkSettings.IPAddress logrus.Infof("[%s] container [%s] is already running on host[%s]", DINDPlane, containerName, cli.DaemonHost()) - return nil + return dindIPAddress, nil } func RmoveDindContainer(ctx context.Context, dindAddress string) error { @@ -88,7 +90,7 @@ func RmoveDindContainer(ctx context.Context, dindAddress string) error { if err != nil { return err } - containerName := DINDContainerPrefix + dindAddress + containerName := fmt.Sprintf("%s-%s", DINDContainerPrefix, dindAddress) logrus.Infof("[%s] Removing dind container [%s] on host [%s]", DINDPlane, containerName, cli.DaemonHost()) _, err = cli.ContainerInspect(ctx, containerName) if err != nil { @@ -104,43 +106,3 @@ func RmoveDindContainer(ctx context.Context, dindAddress string) error { logrus.Infof("[%s] Successfully Removed dind container [%s] on host [%s]", DINDPlane, containerName, cli.DaemonHost()) return nil } - -func CreateDindNetwork(ctx context.Context, dindSubnet string) error { - cli, err := client.NewEnvClient() - if err != nil { - return err - } - networkList, err := cli.NetworkList(ctx, types.NetworkListOptions{}) - for _, net := range networkList { - if DINDNetwork == net.Name { - subnetFound := false - for _, netConfig := range net.IPAM.Config { - if netConfig.Subnet == dindSubnet { - subnetFound = true - break - } - } - if !subnetFound { - return fmt.Errorf("dind network [%s] exist but has different subnet than specified", DINDNetwork) - } - logrus.Infof("[%s] dind network [%s] with subnet [%s] already created", DINDPlane, DINDNetwork, dindSubnet) - return nil - } - } - logrus.Infof("[%s] creating dind network [%s] with subnet [%s]", DINDPlane, DINDNetwork, dindSubnet) - _, err = cli.NetworkCreate(ctx, DINDNetwork, types.NetworkCreate{ - Driver: "bridge", - IPAM: &network.IPAM{ - Config: []network.IPAMConfig{ - network.IPAMConfig{ - Subnet: dindSubnet, - }, - }, - }, - }) - if err != nil { - return err - } - logrus.Infof("[%s] Successfully Created dind network [%s] with subnet [%s]", DINDPlane, DINDNetwork, dindSubnet) - return nil -}