Add local option to deploy/remove kubernetes on local machine

Remove insecure port from kube-api

Use cluster.yml config

Pass config dir to cluster up/remove

cluster/cluster.go

@@ -52,6 +52,9 @@ const (
 	KubeDNSAutoScalerImage     = "kubedns_autoscaler_image"
 	ServiceSidekickImage       = "service_sidekick_image"
 	NoneAuthorizationMode      = "none"
+	LocalNodeAddress           = "127.0.0.1"
+	LocalNodeHostname          = "localhost"
+	LocalNodeUser              = "root"
 )
 
 func (c *Cluster) DeployControlPlane(ctx context.Context) error {
@@ -97,7 +100,12 @@ func ParseConfig(clusterFile string) (*v3.RancherKubernetesEngineConfig, error)
 	return &rkeConfig, nil
 }
 
-func ParseCluster(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig, clusterFilePath string, dockerDialerFactory, localConnDialerFactory hosts.DialerFactory) (*Cluster, error) {
+func ParseCluster(
+	ctx context.Context,
+	rkeConfig *v3.RancherKubernetesEngineConfig,
+	clusterFilePath, configDir string,
+	dockerDialerFactory,
+	localConnDialerFactory hosts.DialerFactory) (*Cluster, error) {
 	var err error
 	c := &Cluster{
 		RancherKubernetesEngineConfig: *rkeConfig,
@@ -126,7 +134,7 @@ func ParseCluster(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConf
 	if len(c.ConfigPath) == 0 {
 		c.ConfigPath = DefaultClusterConfig
 	}
-	c.LocalKubeConfigPath = GetLocalKubeConfig(c.ConfigPath)
+	c.LocalKubeConfigPath = GetLocalKubeConfig(c.ConfigPath, configDir)
 	return c, nil
 }
 
@@ -199,8 +207,11 @@ func (c *Cluster) setClusterImageDefaults() {
 	}
 }
 
-func GetLocalKubeConfig(configPath string) string {
+func GetLocalKubeConfig(configPath, configDir string) string {
 	baseDir := filepath.Dir(configPath)
+	if len(configDir) > 0 {
+		baseDir = filepath.Dir(configDir)
+	}
 	fileName := filepath.Base(configPath)
 	baseDir += "/"
 	return fmt.Sprintf("%s%s%s", baseDir, pki.KubeAdminConfigPrefix, fileName)
@@ -287,3 +298,13 @@ func (c *Cluster) ApplyAuthzResources(ctx context.Context) error {
 	}
 	return nil
 }
+
+func GetLocalRKENodeConfig() *v3.RKEConfigNode {
+	rkeLocalNode := &v3.RKEConfigNode{
+		Address:          LocalNodeAddress,
+		HostnameOverride: LocalNodeHostname,
+		User:             LocalNodeUser,
+		Role:             []string{services.ControlRole, services.WorkerRole, services.ETCDRole},
+	}
+	return rkeLocalNode
+}

cluster/hosts.go

@@ -12,7 +12,13 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func (c *Cluster) TunnelHosts(ctx context.Context) error {
+func (c *Cluster) TunnelHosts(ctx context.Context, local bool) error {
+	if local {
+		if err := c.EtcdHosts[0].TunnelUpLocal(ctx); err != nil {
+			return fmt.Errorf("Failed to connect to docker for local host [%s]: %v", c.EtcdHosts[0].Address, err)
+		}
+		return nil
+	}
 	for i := range c.EtcdHosts {
 		if err := c.EtcdHosts[i].TunnelUp(ctx, c.DockerDialerFactory); err != nil {
 			return fmt.Errorf("Failed to set up SSH tunneling for Etcd host [%s]: %v", c.EtcdHosts[i].Address, err)

cmd/remove.go

@@ -27,6 +27,10 @@ func RemoveCommand() cli.Command {
 			Name:  "force",
 			Usage: "Force removal of the cluster",
 		},
+		cli.BoolFlag{
+			Name:  "local",
+			Usage: "Deploy Kubernetes cluster locally",
+		},
 	}
 	return cli.Command{
 		Name:   "remove",
@@ -36,14 +40,19 @@ func RemoveCommand() cli.Command {
 	}
 }
 
-func ClusterRemove(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig, dialerFactory hosts.DialerFactory) error {
+func ClusterRemove(
-	log.Infof(ctx, "Tearing down Kubernetes cluster")
+	ctx context.Context,
-	kubeCluster, err := cluster.ParseCluster(ctx, rkeConfig, clusterFilePath, dialerFactory, nil)
+	rkeConfig *v3.RancherKubernetesEngineConfig,
+	dialerFactory hosts.DialerFactory,
+	local bool, configDir string) error {
+
+	logrus.Infof("Tearing down Kubernetes cluster")
+	kubeCluster, err := cluster.ParseCluster(ctx, rkeConfig, clusterFilePath, configDir, dialerFactory, nil)
 	if err != nil {
 		return err
 	}
 
-	err = kubeCluster.TunnelHosts(ctx)
+	err = kubeCluster.TunnelHosts(ctx, local)
 	if err != nil {
 		return err
 	}
@@ -59,6 +68,7 @@ func ClusterRemove(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineCon
 }
 
 func clusterRemoveFromCli(ctx *cli.Context) error {
+	var local bool
 	force := ctx.Bool("force")
 	if !force {
 		reader := bufio.NewReader(os.Stdin)
@@ -77,10 +87,13 @@ func clusterRemoveFromCli(ctx *cli.Context) error {
 		return fmt.Errorf("Failed to resolve cluster file: %v", err)
 	}
 	clusterFilePath = filePath
-
 	rkeConfig, err := cluster.ParseConfig(clusterFile)
 	if err != nil {
 		return fmt.Errorf("Failed to parse cluster file: %v", err)
 	}
-	return ClusterRemove(context.Background(), rkeConfig, nil)
+	if ctx.Bool("local") {
+		rkeConfig.Nodes = []v3.RKEConfigNode{*cluster.GetLocalRKENodeConfig()}
+		local = true
+	}
+	return ClusterRemove(context.Background(), rkeConfig, nil, local, "")
 }

cmd/up.go

@@ -23,6 +23,10 @@ func UpCommand() cli.Command {
 			Value:  cluster.DefaultClusterConfig,
 			EnvVar: "RKE_CONFIG",
 		},
+		cli.BoolFlag{
+			Name:  "local",
+			Usage: "Deploy Kubernetes cluster locally",
+		},
 	}
 	return cli.Command{
 		Name:   "up",
@@ -32,15 +36,20 @@ func UpCommand() cli.Command {
 	}
 }
 
-func ClusterUp(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig, dockerDialerFactory, localConnDialerFactory hosts.DialerFactory) (string, string, string, string, error) {
+func ClusterUp(
+	ctx context.Context,
+	rkeConfig *v3.RancherKubernetesEngineConfig,
+	dockerDialerFactory, localConnDialerFactory hosts.DialerFactory,
+	local bool, configDir string) (string, string, string, string, error) {
+
 	log.Infof(ctx, "Building Kubernetes cluster")
 	var APIURL, caCrt, clientCert, clientKey string
-	kubeCluster, err := cluster.ParseCluster(ctx, rkeConfig, clusterFilePath, dockerDialerFactory, localConnDialerFactory)
+	kubeCluster, err := cluster.ParseCluster(ctx, rkeConfig, clusterFilePath, configDir, dockerDialerFactory, localConnDialerFactory)
 	if err != nil {
 		return APIURL, caCrt, clientCert, clientKey, err
 	}
 
-	err = kubeCluster.TunnelHosts(ctx)
+	err = kubeCluster.TunnelHosts(ctx, local)
 	if err != nil {
 		return APIURL, caCrt, clientCert, clientKey, err
 	}
@@ -105,6 +114,8 @@ func ClusterUp(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig,
 }
 
 func clusterUpFromCli(ctx *cli.Context) error {
+	var local bool
+	var localConnDialerFactory hosts.DialerFactory
 	clusterFile, filePath, err := resolveClusterFile(ctx)
 	if err != nil {
 		return fmt.Errorf("Failed to resolve cluster file: %v", err)
@@ -115,6 +126,11 @@ func clusterUpFromCli(ctx *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("Failed to parse cluster file: %v", err)
 	}
-	_, _, _, _, err = ClusterUp(context.Background(), rkeConfig, nil, nil)
+	if ctx.Bool("local") {
+		rkeConfig.Nodes = []v3.RKEConfigNode{*cluster.GetLocalRKENodeConfig()}
+		localConnDialerFactory = hosts.LocalHealthcheckFactory
+		local = true
+	}
+	_, _, _, _, err = ClusterUp(context.Background(), rkeConfig, nil, localConnDialerFactory, local, "")
 	return err
 }

cmd/version.go

@@ -25,7 +25,7 @@ func VersionCommand() cli.Command {
 }
 
 func getClusterVersion(ctx *cli.Context) error {
-	localKubeConfig := cluster.GetLocalKubeConfig(ctx.String("config"))
+	localKubeConfig := cluster.GetLocalKubeConfig(ctx.String("config"), "")
 	serverVersion, err := cluster.GetK8sVersion(localKubeConfig)
 	if err != nil {
 		return err

hosts/local.go

@@ -0,0 +1,21 @@
+package hosts
+
+import (
+	"fmt"
+	"net"
+)
+
+func LocalHealthcheckFactory(h *Host) (func(network, address string) (net.Conn, error), error) {
+	dialer := &dialer{
+		host: h,
+	}
+	return dialer.DialHealthcheckLocally, nil
+}
+
+func (d *dialer) DialHealthcheckLocally(network, addr string) (net.Conn, error) {
+	conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%d", d.host.LocalConnPort))
+	if err != nil {
+		return nil, fmt.Errorf("Failed to dial address [%s]: %v", d.host.Address, err)
+	}
+	return conn, err
+}

hosts/tunnel.go

@@ -31,13 +31,30 @@ func (h *Host) TunnelUp(ctx context.Context, dialerFactory DialerFactory) error
 	if err != nil {
 		return fmt.Errorf("Can't establish dialer connection: %v", err)
 	}
-
 	// set Docker client
 	logrus.Debugf("Connecting to Docker API for host [%s]", h.Address)
 	h.DClient, err = client.NewClient("unix:///var/run/docker.sock", DockerAPIVersion, httpClient, nil)
 	if err != nil {
 		return fmt.Errorf("Can't initiate NewClient: %v", err)
 	}
+	return checkDockerVersion(ctx, h)
+}
+
+func (h *Host) TunnelUpLocal(ctx context.Context) error {
+	var err error
+	if h.DClient != nil {
+		return nil
+	}
+	// set Docker client
+	logrus.Debugf("Connecting to Docker API for host [%s]", h.Address)
+	h.DClient, err = client.NewEnvClient()
+	if err != nil {
+		return fmt.Errorf("Can't initiate NewClient: %v", err)
+	}
+	return checkDockerVersion(ctx, h)
+}
+
+func checkDockerVersion(ctx context.Context, h *Host) error {
 	info, err := h.DClient.Info(ctx)
 	if err != nil {
 		return fmt.Errorf("Can't retrieve Docker Info: %v", err)
@@ -53,7 +70,6 @@ func (h *Host) TunnelUp(ctx context.Context, dialerFactory DialerFactory) error
 	} else if !isvalid {
 		log.Warnf(ctx, "Unsupported Docker version found [%s], supported versions are %v", info.ServerVersion, docker.K8sDockerVersions[K8sVersion])
 	}
-
 	return nil
 }
 

services/kubeapi.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 
 	"github.com/docker/docker/api/types/container"
-	"github.com/docker/go-connections/nat"
 	"github.com/rancher/rke/docker"
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/pki"
@@ -18,7 +17,7 @@ func runKubeAPI(ctx context.Context, host *hosts.Host, etcdHosts []*hosts.Host,
 	if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, KubeAPIContainerName, host.Address, ControlRole); err != nil {
 		return err
 	}
-	return runHealthcheck(ctx, host, KubeAPIPort, false, KubeAPIContainerName, df)
+	return runHealthcheck(ctx, host, KubeAPIPort, true, KubeAPIContainerName, df)
 }
 
 func removeKubeAPI(ctx context.Context, host *hosts.Host) error {
@@ -32,7 +31,7 @@ func buildKubeAPIConfig(host *hosts.Host, kubeAPIService v3.KubeAPIService, etcd
 			"kube-apiserver",
 			"--insecure-bind-address=127.0.0.1",
 			"--bind-address=0.0.0.0",
-			"--insecure-port=8080",
+			"--insecure-port=0",
 			"--secure-port=6443",
 			"--cloud-provider=",
 			"--allow_privileged=true",
@@ -63,14 +62,6 @@ func buildKubeAPIConfig(host *hosts.Host, kubeAPIService v3.KubeAPIService, etcd
 		},
 		NetworkMode:   "host",
 		RestartPolicy: container.RestartPolicy{Name: "always"},
-		PortBindings: nat.PortMap{
-			"8080/tcp": []nat.PortBinding{
-				{
-					HostIP:   "0.0.0.0",
-					HostPort: "8080",
-				},
-			},
-		},
 	}
 
 	for arg, value := range kubeAPIService.ExtraArgs {

services/services.go

@@ -28,7 +28,7 @@ const (
 	NginxProxyContainerName     = "nginx-proxy"
 	SidekickContainerName       = "service-sidekick"
 
-	KubeAPIPort        = 8080
+	KubeAPIPort        = 6443
 	SchedulerPort      = 10251
 	KubeControllerPort = 10252
 	KubeletPort        = 10250