Add executor sandbox overlay mechanism to distribute nsenter and socat

cluster/mesos/docker/docker-compose.yml

@@ -147,6 +147,7 @@ scheduler:
     --cluster-dns=10.10.10.10
     --cluster-domain=cluster.local
     --mesos-executor-cpus=1.0
+    --mesos-sandbox-overlay=/opt/sandbox-overlay.tar.gz
     --v=4
     --executor-logv=4
     --profiling=true

cluster/mesos/docker/km/Dockerfile

@@ -14,4 +14,4 @@ RUN apt-get update -qq && \
     apt-get clean
 
 COPY ./bin/* /usr/local/bin/
-ADD ./opt/mesos-cloud.conf /opt/
+COPY ./opt/* /opt/

cluster/mesos/docker/km/build.sh

@@ -47,6 +47,11 @@ fi
 kube_bin_path=$(dirname ${km_path})
 common_bin_path=$(cd ${script_dir}/../common/bin && pwd -P)
 
+# download nsenter and socat
+mkdir -p "${script_dir}/overlay"
+docker run --rm -v "${script_dir}/overlay:/target" jpetazzo/nsenter
+docker run --rm -v "${script_dir}/overlay:/target" mesosphere/kubernetes-socat
+
 cd "${KUBE_ROOT}"
 
 # create temp workspace to place compiled binaries with image-specific scripts
@@ -65,6 +70,7 @@ echo "Copying files to workspace"
 
 # binaries & scripts
 mkdir -p "${workspace}/bin"
+
 #cp "${script_dir}/bin/"* "${workspace}/bin/"
 cp "${common_bin_path}/"* "${workspace}/bin/"
 cp "${kube_bin_path}/km" "${workspace}/bin/"
@@ -73,6 +79,13 @@ cp "${kube_bin_path}/km" "${workspace}/bin/"
 mkdir -p "${workspace}/opt"
 cp "${script_dir}/opt/"* "${workspace}/opt/"
 
+# package up the sandbox overay
+mkdir -p "${workspace}/overlay/bin"
+cp -a "${script_dir}/overlay/nsenter" "${workspace}/overlay/bin"
+cp -a "${script_dir}/overlay/socat" "${workspace}/overlay/bin"
+chmod +x "${workspace}/overlay/bin/"*
+cd "${workspace}/overlay" && tar -czvf "${workspace}/opt/sandbox-overlay.tar.gz" . && cd -
+
 # docker
 cp "${script_dir}/Dockerfile" "${workspace}/"
 

cluster/mesos/docker/socat/Dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:14.04.3
+MAINTAINER Mesosphere <support@mesosphere.io>
+
+RUN apt-get update -qq && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qqy \
+        build-essential curl \
+        && \
+    apt-get clean
+
+RUN mkdir -p /src
+WORKDIR /src
+RUN curl -f -osocat-1.7.2.4.tar.bz2 http://www.dest-unreach.org/socat/download/socat-1.7.2.4.tar.bz2
+RUN tar -xjvf socat-1.7.2.4.tar.bz2 && cd socat-1.7.2.4 && ./configure --disable-openssl && LDFLAGS=-static make
+
+VOLUME ["/target"]
+CMD ["cp", "/src/socat-1.7.2.4/socat", "/target"]

cluster/mesos/docker/socat/build.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Builds a docker image that contains the kubernetes-mesos binaries.
+
+set -o errexit
+set -o nounset
+set -o pipefailscript_dir=$(cd $(dirname "${BASH_SOURCE}") && pwd -P)
+
+cd "${script_dir}"
+
+docker build -t mesosphere/kubernetes-socat .

contrib/mesos/pkg/minion/server.go

@@ -206,17 +206,23 @@ func (ms *MinionServer) launchHyperkubeServer(server string, args []string, logF
 		}
 	}
 
-	// use given environment, but add /usr/sbin to the path for the iptables binary used in kube-proxy
+	// use given environment, but add /usr/sbin and $SANDBOX/bin to the path for the iptables binary used in kube-proxy
 	var kmEnv []string
-	if ms.pathOverride != "" {
 	env := os.Environ()
 	kmEnv = make([]string, 0, len(env))
 	for _, e := range env {
 		if !strings.HasPrefix(e, "PATH=") {
 			kmEnv = append(kmEnv, e)
+		} else {
+			if ms.pathOverride != "" {
+				e = "PATH=" + ms.pathOverride
 			}
+			pwd, err := os.Getwd()
+			if err != nil {
+				log.Fatalf("Cannot get current directory: %v", err)
+			}
+			kmEnv = append(kmEnv, fmt.Sprintf("%s:%s", e, path.Join(pwd, "bin")))
 		}
-		kmEnv = append(kmEnv, "PATH="+ms.pathOverride)
 	}
 
 	t := tasks.New(server, ms.kmBinary, kmArgs, kmEnv, writerFunc)

contrib/mesos/pkg/scheduler/service/service.go

@@ -27,6 +27,7 @@ import (
 	"os"
 	"os/exec"
 	"os/user"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
@@ -150,6 +151,7 @@ type SchedulerServer struct {
 	ContainPodResources           bool
 	AccountForPodResources        bool
 	nodeRelistPeriod              time.Duration
+	SandboxOverlay                string
 
 	executable  string // path to the binary running this service
 	client      *client.Client
@@ -258,6 +260,7 @@ func (s *SchedulerServer) addCoreFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&s.ExecutorBindall, "executor-bindall", s.ExecutorBindall, "When true will set -address of the executor to 0.0.0.0.")
 	fs.DurationVar(&s.ExecutorSuicideTimeout, "executor-suicide-timeout", s.ExecutorSuicideTimeout, "Executor self-terminates after this period of inactivity. Zero disables suicide watch.")
 	fs.DurationVar(&s.LaunchGracePeriod, "mesos-launch-grace-period", s.LaunchGracePeriod, "Launch grace period after which launching tasks will be cancelled. Zero disables launch cancellation.")
+	fs.StringVar(&s.SandboxOverlay, "mesos-sandbox-overlay", s.SandboxOverlay, "Path to an archive extracted in the sandbox.")
 
 	fs.BoolVar(&s.ProxyBindall, "proxy-bindall", s.ProxyBindall, "When true pass -proxy-bindall to the executor.")
 	fs.BoolVar(&s.RunProxy, "run-proxy", s.RunProxy, "Run the kube-proxy as a side process of the executor.")
@@ -366,6 +369,11 @@ func (s *SchedulerServer) prepareExecutorInfo(hks hyperkube.Interface) (*mesos.E
 		ci.Arguments = append(ci.Arguments, fmt.Sprintf("--max-log-age=%d", s.MinionLogMaxAgeInDays))
 	}
 
+	if s.SandboxOverlay != "" {
+		uri, _ := s.serveFrameworkArtifact(s.SandboxOverlay)
+		ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(false), Extract: proto.Bool(true)})
+	}
+
 	if s.DockerCfgPath != "" {
 		uri := s.serveFrameworkArtifactWithFilename(s.DockerCfgPath, ".dockercfg")
 		ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(false), Extract: proto.Bool(false)})

hack/verify-flags/known-flags.txt

@@ -178,10 +178,11 @@ mesos-authentication-secret-file
 mesos-cgroup-prefix
 mesos-executor-cpus
 mesos-executor-mem
+mesos-launch-grace-period
 mesos-master
 mesos-role
+mesos-sandbox-overlay
 mesos-user
-mesos-launch-grace-period
 minimum-container-ttl-duration
 minion-max-log-age
 minion-max-log-backups
@@ -308,4 +309,3 @@ terminated-pod-gc-threshold
 reconcile-cidr
 register-schedulable
 repair-malformed-updates
-