Merge pull request #12872 from runningwild/master

Fix several problems with using rkt on gce

cluster/gce/coreos/helper.sh

@@ -67,35 +67,35 @@ KUBELET_APISERVER: $(yaml-quote ${KUBELET_APISERVER:-})
 EOF
   else
     cat >>$file <<EOF
-KUBERNETES_MASTER: "false"
+KUBERNETES_MASTER="false"
-ENV_TIMESTAMP: $(yaml-quote $(date -u +%Y-%m-%dT%T%z))
+ENV_TIMESTAMP=$(date -u +%Y-%m-%dT%T%z)
-INSTANCE_PREFIX: $(yaml-quote ${INSTANCE_PREFIX})
+INSTANCE_PREFIX=${INSTANCE_PREFIX}
-NODE_INSTANCE_PREFIX: $(yaml-quote ${NODE_INSTANCE_PREFIX})
+NODE_INSTANCE_PREFIX=${NODE_INSTANCE_PREFIX}
-SERVER_BINARY_TAR_URL: $(yaml-quote ${SERVER_BINARY_TAR_URL})
+SERVER_BINARY_TAR_URL=${SERVER_BINARY_TAR_URL}
-SERVICE_CLUSTER_IP_RANGE: $(yaml-quote ${SERVICE_CLUSTER_IP_RANGE})
+SERVICE_CLUSTER_IP_RANGE=${SERVICE_CLUSTER_IP_RANGE}
-ENABLE_CLUSTER_MONITORING: $(yaml-quote ${ENABLE_CLUSTER_MONITORING:-none})
+ENABLE_CLUSTER_MONITORING=${ENABLE_CLUSTER_MONITORING:-none}
-ENABLE_CLUSTER_LOGGING: $(yaml-quote ${ENABLE_CLUSTER_LOGGING:-false})
+ENABLE_CLUSTER_LOGGING=${ENABLE_CLUSTER_LOGGING:-false}
-ENABLE_CLUSTER_UI: $(yaml-quote ${ENABLE_CLUSTER_UI:-false})
+ENABLE_CLUSTER_UI=${ENABLE_CLUSTER_UI:-false}
-ENABLE_NODE_LOGGING: $(yaml-quote ${ENABLE_NODE_LOGGING:-false})
+ENABLE_NODE_LOGGING=${ENABLE_NODE_LOGGING:-false}
-LOGGING_DESTINATION: $(yaml-quote ${LOGGING_DESTINATION:-})
+LOGGING_DESTINATION=${LOGGING_DESTINATION:-}
-ELASTICSEARCH_LOGGING_REPLICAS: $(yaml-quote ${ELASTICSEARCH_LOGGING_REPLICAS:-})
+ELASTICSEARCH_LOGGING_REPLICAS=${ELASTICSEARCH_LOGGING_REPLICAS:-}
-ENABLE_CLUSTER_DNS: $(yaml-quote ${ENABLE_CLUSTER_DNS:-false})
+ENABLE_CLUSTER_DNS=${ENABLE_CLUSTER_DNS:-false}
-DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-})
+DNS_REPLICAS=${DNS_REPLICAS:-}
-DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-})
+DNS_SERVER_IP=${DNS_SERVER_IP:-}
-DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-})
+DNS_DOMAIN=${DNS_DOMAIN:-}
-KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-})
+KUBELET_TOKEN=${KUBELET_TOKEN:-}
-KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-})
+KUBE_PROXY_TOKEN=${KUBE_PROXY_TOKEN:-}
-ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-})
+ADMISSION_CONTROL=${ADMISSION_CONTROL:-}
-MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE})
+MASTER_IP_RANGE=${MASTER_IP_RANGE}
-KUBERNETES_MASTER_NAME: $(yaml-quote ${MASTER_NAME})
+KUBERNETES_MASTER_NAME=${MASTER_NAME}
-ZONE: $(yaml-quote ${ZONE})
+ZONE=${ZONE}
-EXTRA_DOCKER_OPTS: $(yaml-quote ${EXTRA_DOCKER_OPTS:-})
+EXTRA_DOCKER_OPTS=${EXTRA_DOCKER_OPTS:-}
-PROJECT_ID: $(yaml-quote ${PROJECT})
+PROJECT_ID=${PROJECT}
-KUBERNETES_CONTAINER_RUNTIME: $(yaml-quote ${CONTAINER_RUNTIME})
+KUBERNETES_CONTAINER_RUNTIME=${CONTAINER_RUNTIME}
-RKT_VERSION: $(yaml-quote ${RKT_VERSION})
+RKT_VERSION=${RKT_VERSION}
-CA_CERT: $(yaml-quote ${CA_CERT_BASE64})
+CA_CERT=${CA_CERT_BASE64}
-KUBELET_CERT: $(yaml-quote ${KUBELET_CERT_BASE64:-})
+KUBELET_CERT=${KUBELET_CERT_BASE64:-}
-KUBELET_KEY: $(yaml-quote ${KUBELET_KEY_BASE64:-})
+KUBELET_KEY=${KUBELET_KEY_BASE64:-}
 EOF
   fi
 }

cluster/gce/coreos/node.yaml

@@ -151,7 +151,8 @@ coreos:
         --cluster-dns=10.0.0.10 \
         --cluster-domain=cluster.local \
         --logtostderr=true \
-        --container-runtime=${KUBERNETES_CONTAINER_RUNTIME}
+        --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \
+        --rkt-path=/opt/rkt/rkt/rkt
         Restart=always
         RestartSec=10
 
@@ -202,4 +203,4 @@ coreos:
         After=kubernetes-install-rkt.service
         [Service]
         EnvironmentFile=/etc/kube-env
-        ExecStart=/opt/rkt/rkt metadata-service
+        ExecStart=/opt/rkt/rkt/rkt metadata-service

cmd/kubelet/app/server.go

@@ -116,6 +116,7 @@ type KubeletServer struct {
 	ResourceContainer              string
 	CgroupRoot                     string
 	ContainerRuntime               string
+	RktPath                        string
 	DockerDaemonContainer          string
 	SystemContainer                string
 	ConfigureCBR0                  bool
@@ -182,6 +183,7 @@ func NewKubeletServer() *KubeletServer {
 		ResourceContainer:           "/kubelet",
 		CgroupRoot:                  "",
 		ContainerRuntime:            "docker",
+		RktPath:                     "",
 		DockerDaemonContainer:       "/docker-daemon",
 		SystemContainer:             "",
 		ConfigureCBR0:               false,
@@ -245,6 +247,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.ResourceContainer, "resource-container", s.ResourceContainer, "Absolute name of the resource-only container to create and run the Kubelet in (Default: /kubelet).")
 	fs.StringVar(&s.CgroupRoot, "cgroup-root", s.CgroupRoot, "Optional root cgroup to use for pods. This is handled by the container runtime on a best effort basis. Default: '', which means use the container runtime default.")
 	fs.StringVar(&s.ContainerRuntime, "container-runtime", s.ContainerRuntime, "The container runtime to use. Possible values: 'docker', 'rkt'. Default: 'docker'.")
+	fs.StringVar(&s.RktPath, "rkt-path", s.RktPath, "Path of rkt binary. Leave empty to use the first rkt in $PATH.  Only used if --container-runtime='rkt'")
 	fs.StringVar(&s.SystemContainer, "system-container", s.SystemContainer, "Optional resource-only container in which to place all non-kernel processes that are not already in a container. Empty for no container. Rolling back the flag requires a reboot. (Default: \"\").")
 	fs.BoolVar(&s.ConfigureCBR0, "configure-cbr0", s.ConfigureCBR0, "If true, kubelet will configure cbr0 based on Node.Spec.PodCIDR.")
 	fs.IntVar(&s.MaxPods, "max-pods", 40, "Number of Pods that can run on this Kubelet.")
@@ -348,6 +351,7 @@ func (s *KubeletServer) KubeletConfig() (*KubeletConfig, error) {
 		ResourceContainer:         s.ResourceContainer,
 		CgroupRoot:                s.CgroupRoot,
 		ContainerRuntime:          s.ContainerRuntime,
+		RktPath:                   s.RktPath,
 		Mounter:                   mounter,
 		DockerDaemonContainer:     s.DockerDaemonContainer,
 		SystemContainer:           s.SystemContainer,
@@ -753,6 +757,7 @@ type KubeletConfig struct {
 	OSInterface                    kubecontainer.OSInterface
 	CgroupRoot                     string
 	ContainerRuntime               string
+	RktPath                        string
 	Mounter                        mount.Interface
 	DockerDaemonContainer          string
 	SystemContainer                string
@@ -811,6 +816,7 @@ func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
 		kc.OSInterface,
 		kc.CgroupRoot,
 		kc.ContainerRuntime,
+		kc.RktPath,
 		kc.Mounter,
 		kc.DockerDaemonContainer,
 		kc.SystemContainer,

contrib/mesos/pkg/executor/service/service.go

@@ -354,6 +354,7 @@ func (ks *KubeletExecutorServer) createAndInitKubelet(
 		kc.OSInterface,
 		kc.CgroupRoot,
 		kc.ContainerRuntime,
+		kc.RktPath,
 		kc.Mounter,
 		kc.DockerDaemonContainer,
 		kc.SystemContainer,

hack/verify-flags/known-flags.txt

@@ -198,6 +198,7 @@ required-contexts
 resource-container
 resource-quota-sync-period
 resource-version
+rkt-path
 root-ca-file
 root-dir
 run-proxy

pkg/kubelet/kubelet.go

@@ -144,6 +144,7 @@ func NewMainKubelet(
 	osInterface kubecontainer.OSInterface,
 	cgroupRoot string,
 	containerRuntime string,
+	rktPath string,
 	mounter mount.Interface,
 	dockerDaemonContainer string,
 	systemContainer string,
@@ -304,7 +305,10 @@ func NewMainKubelet(
 			oomAdjuster,
 			procFs)
 	case "rkt":
-		conf := &rkt.Config{InsecureSkipVerify: true}
+		conf := &rkt.Config{
+			Path:               rktPath,
+			InsecureSkipVerify: true,
+		}
 		rktRuntime, err := rkt.New(
 			conf,
 			klet,

pkg/kubelet/rkt/config.go

@@ -21,6 +21,8 @@ import "fmt"
 // Config stores the global configuration for the rkt runtime.
 // Run 'rkt' for more details.
 type Config struct {
+	// The absolute path to the binary, or leave empty to find it in $PATH.
+	Path string
 	// The debug flag for rkt.
 	Debug bool
 	// The rkt data directory.

pkg/kubelet/rkt/rkt.go

@@ -131,11 +131,14 @@ func New(config *Config,
 		return nil, fmt.Errorf("cannot connect to dbus: %v", err)
 	}
 
-	// Test if rkt binary is in $PATH.
+	rktBinAbsPath := config.Path
-	// TODO(yifan): Use a kubelet flag to read the path.
+	if rktBinAbsPath == "" {
-	rktBinAbsPath, err := exec.LookPath("rkt")
+		// No default rkt path was set, so try to find one in $PATH.
-	if err != nil {
+		var err error
-		return nil, fmt.Errorf("cannot find rkt binary: %v", err)
+		rktBinAbsPath, err = exec.LookPath("rkt")
+		if err != nil {
+			return nil, fmt.Errorf("cannot find rkt binary: %v", err)
+		}
 	}
 
 	rkt := &runtime{