diff --git a/cmd/up.go b/cmd/up.go index fe8e7f51..3bb3accf 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -40,6 +40,11 @@ func UpCommand() cli.Command { Name: "dind-storage-driver", Usage: "Storage driver for the docker in docker containers (experimental)", }, + cli.StringFlag{ + Name: "dind-dns-server", + Usage: "DNS resolver to be used by docker in docker container. Useful if host is running systemd-resovld", + Value: "8.8.8.8", + }, cli.BoolFlag{ Name: "update-only", Usage: "Skip idempotent deployment of control and etcd plane", @@ -312,12 +317,12 @@ func clusterUpLocal(ctx *cli.Context) error { func clusterUpDind(ctx *cli.Context) error { // get dind config - rkeConfig, disablePortCheck, dindStorageDriver, filePath, err := getDindConfig(ctx) + rkeConfig, disablePortCheck, dindStorageDriver, filePath, dindDNS, err := getDindConfig(ctx) if err != nil { return err } // setup dind environment - if err = createDINDEnv(context.Background(), rkeConfig, dindStorageDriver); err != nil { + if err = createDINDEnv(context.Background(), rkeConfig, dindStorageDriver, dindDNS); err != nil { return err } @@ -338,23 +343,24 @@ func clusterUpDind(ctx *cli.Context) error { return err } -func getDindConfig(ctx *cli.Context) (*v3.RancherKubernetesEngineConfig, bool, string, string, error) { +func getDindConfig(ctx *cli.Context) (*v3.RancherKubernetesEngineConfig, bool, string, string, string, error) { disablePortCheck := ctx.Bool("disable-port-check") dindStorageDriver := ctx.String("dind-storage-driver") + dindDNS := ctx.String("dind-dns-server") clusterFile, filePath, err := resolveClusterFile(ctx) if err != nil { - return nil, disablePortCheck, "", "", fmt.Errorf("Failed to resolve cluster file: %v", err) + return nil, disablePortCheck, "", "", "", fmt.Errorf("Failed to resolve cluster file: %v", err) } rkeConfig, err := cluster.ParseConfig(clusterFile) if err != nil { - return nil, disablePortCheck, "", "", fmt.Errorf("Failed to parse cluster file: %v", err) + return nil, disablePortCheck, "", "", "", fmt.Errorf("Failed to parse cluster file: %v", err) } rkeConfig, err = setOptionsFromCLI(ctx, rkeConfig) if err != nil { - return nil, disablePortCheck, "", "", err + return nil, disablePortCheck, "", "", "", err } // Setting conntrack max for kubeproxy to 0 if rkeConfig.Services.Kubeproxy.ExtraArgs == nil { @@ -362,12 +368,12 @@ func getDindConfig(ctx *cli.Context) (*v3.RancherKubernetesEngineConfig, bool, s } rkeConfig.Services.Kubeproxy.ExtraArgs["conntrack-max-per-core"] = "0" - return rkeConfig, disablePortCheck, dindStorageDriver, filePath, nil + return rkeConfig, disablePortCheck, dindStorageDriver, filePath, dindDNS, nil } -func createDINDEnv(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig, dindStorageDriver string) error { +func createDINDEnv(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig, dindStorageDriver, dindDNS string) error { for i := range rkeConfig.Nodes { - address, err := dind.StartUpDindContainer(ctx, rkeConfig.Nodes[i].Address, dind.DINDNetwork, dindStorageDriver) + address, err := dind.StartUpDindContainer(ctx, rkeConfig.Nodes[i].Address, dind.DINDNetwork, dindStorageDriver, dindDNS) if err != nil { return err } diff --git a/dind/dind.go b/dind/dind.go index 2afe58f6..a2eb5df6 100644 --- a/dind/dind.go +++ b/dind/dind.go @@ -8,6 +8,7 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/docker/client" "github.com/rancher/rke/docker" + "github.com/rancher/rke/util" "github.com/sirupsen/logrus" ) @@ -19,7 +20,7 @@ const ( DINDSubnet = "172.18.0.0/16" ) -func StartUpDindContainer(ctx context.Context, dindAddress, dindNetwork, dindStorageDriver string) (string, error) { +func StartUpDindContainer(ctx context.Context, dindAddress, dindNetwork, dindStorageDriver, dindDNS string) (string, error) { cli, err := client.NewEnvClient() if err != nil { return "", err @@ -46,7 +47,15 @@ func StartUpDindContainer(ctx context.Context, dindAddress, dindNetwork, dindSto } binds := []string{ fmt.Sprintf("/var/lib/kubelet-%s:/var/lib/kubelet:shared", containerName), - "/etc/resolv.conf:/etc/resolv.conf", + } + isLink, err := util.IsSymlink("/etc/resolv.conf") + if err != nil { + return "", err + } + if isLink { + logrus.Infof("[%s] symlinked [/etc/resolv.conf] file detected. Using [%s] as DNS server.", DINDPlane, dindDNS) + } else { + binds = append(binds, "/etc/resolv.conf:/etc/resolv.conf") } imageCfg := &container.Config{ Image: DINDImage, @@ -62,6 +71,12 @@ func StartUpDindContainer(ctx context.Context, dindAddress, dindNetwork, dindSto hostCfg := &container.HostConfig{ Privileged: true, Binds: binds, + // this gets ignored if resolv.conf is bind mounted. So it's ok to have it anyway. + DNS: []string{dindDNS}, + // Calico needs this + Sysctls: map[string]string{ + "net.ipv4.conf.all.rp_filter": "1", + }, } resp, err := cli.ContainerCreate(ctx, imageCfg, hostCfg, nil, containerName) if err != nil { diff --git a/util/util.go b/util/util.go index 03db3897..cd054ae7 100644 --- a/util/util.go +++ b/util/util.go @@ -2,6 +2,7 @@ package util import ( "fmt" + "os" "reflect" "strings" @@ -51,3 +52,14 @@ func UniqueStringSlice(elements []string) []string { } return result } + +func IsSymlink(file string) (bool, error) { + f, err := os.Lstat(file) + if err != nil { + return false, err + } + if f.Mode()&os.ModeSymlink != 0 { + return true, nil + } + return false, nil +}