From 4062e2be2fba1b1ce609314279efdaa115386bda Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 22 Oct 2015 11:03:22 -0700 Subject: [PATCH 1/5] Add executor sandbox overlay mechanism to distribute nsenter and socat --- cluster/mesos/docker/docker-compose.yml | 1 + cluster/mesos/docker/km/Dockerfile | 2 +- cluster/mesos/docker/km/build.sh | 13 ++++++++++ cluster/mesos/docker/socat/Dockerfile | 16 ++++++++++++ cluster/mesos/docker/socat/build.sh | 25 +++++++++++++++++++ contrib/mesos/pkg/minion/server.go | 22 ++++++++++------ .../mesos/pkg/scheduler/service/service.go | 8 ++++++ hack/verify-flags/known-flags.txt | 4 +-- 8 files changed, 80 insertions(+), 11 deletions(-) create mode 100644 cluster/mesos/docker/socat/Dockerfile create mode 100755 cluster/mesos/docker/socat/build.sh diff --git a/cluster/mesos/docker/docker-compose.yml b/cluster/mesos/docker/docker-compose.yml index 0aa480dd8de..63516712311 100644 --- a/cluster/mesos/docker/docker-compose.yml +++ b/cluster/mesos/docker/docker-compose.yml @@ -147,6 +147,7 @@ scheduler: --cluster-dns=10.10.10.10 --cluster-domain=cluster.local --mesos-executor-cpus=1.0 + --mesos-sandbox-overlay=/opt/sandbox-overlay.tar.gz --v=4 --executor-logv=4 --profiling=true diff --git a/cluster/mesos/docker/km/Dockerfile b/cluster/mesos/docker/km/Dockerfile index 2bbb7c338a3..857153c23a2 100644 --- a/cluster/mesos/docker/km/Dockerfile +++ b/cluster/mesos/docker/km/Dockerfile @@ -14,4 +14,4 @@ RUN apt-get update -qq && \ apt-get clean COPY ./bin/* /usr/local/bin/ -ADD ./opt/mesos-cloud.conf /opt/ +COPY ./opt/* /opt/ diff --git a/cluster/mesos/docker/km/build.sh b/cluster/mesos/docker/km/build.sh index fe62a2b3085..cc4f38d6eb8 100755 --- a/cluster/mesos/docker/km/build.sh +++ b/cluster/mesos/docker/km/build.sh @@ -47,6 +47,11 @@ fi kube_bin_path=$(dirname ${km_path}) common_bin_path=$(cd ${script_dir}/../common/bin && pwd -P) +# download nsenter and socat +mkdir -p "${script_dir}/overlay" +docker run --rm -v "${script_dir}/overlay:/target" jpetazzo/nsenter +docker run --rm -v "${script_dir}/overlay:/target" mesosphere/kubernetes-socat + cd "${KUBE_ROOT}" # create temp workspace to place compiled binaries with image-specific scripts @@ -65,6 +70,7 @@ echo "Copying files to workspace" # binaries & scripts mkdir -p "${workspace}/bin" + #cp "${script_dir}/bin/"* "${workspace}/bin/" cp "${common_bin_path}/"* "${workspace}/bin/" cp "${kube_bin_path}/km" "${workspace}/bin/" @@ -73,6 +79,13 @@ cp "${kube_bin_path}/km" "${workspace}/bin/" mkdir -p "${workspace}/opt" cp "${script_dir}/opt/"* "${workspace}/opt/" +# package up the sandbox overay +mkdir -p "${workspace}/overlay/bin" +cp -a "${script_dir}/overlay/nsenter" "${workspace}/overlay/bin" +cp -a "${script_dir}/overlay/socat" "${workspace}/overlay/bin" +chmod +x "${workspace}/overlay/bin/"* +cd "${workspace}/overlay" && tar -czvf "${workspace}/opt/sandbox-overlay.tar.gz" . && cd - + # docker cp "${script_dir}/Dockerfile" "${workspace}/" diff --git a/cluster/mesos/docker/socat/Dockerfile b/cluster/mesos/docker/socat/Dockerfile new file mode 100644 index 00000000000..e0f302b3108 --- /dev/null +++ b/cluster/mesos/docker/socat/Dockerfile @@ -0,0 +1,16 @@ +FROM ubuntu:14.04.3 +MAINTAINER Mesosphere + +RUN apt-get update -qq && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qqy \ + build-essential curl \ + && \ + apt-get clean + +RUN mkdir -p /src +WORKDIR /src +RUN curl -f -osocat-1.7.2.4.tar.bz2 http://www.dest-unreach.org/socat/download/socat-1.7.2.4.tar.bz2 +RUN tar -xjvf socat-1.7.2.4.tar.bz2 && cd socat-1.7.2.4 && ./configure --disable-openssl && LDFLAGS=-static make + +VOLUME ["/target"] +CMD ["cp", "/src/socat-1.7.2.4/socat", "/target"] diff --git a/cluster/mesos/docker/socat/build.sh b/cluster/mesos/docker/socat/build.sh new file mode 100755 index 00000000000..f001d31cc39 --- /dev/null +++ b/cluster/mesos/docker/socat/build.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds a docker image that contains the kubernetes-mesos binaries. + +set -o errexit +set -o nounset +set -o pipefailscript_dir=$(cd $(dirname "${BASH_SOURCE}") && pwd -P) + +cd "${script_dir}" + +docker build -t mesosphere/kubernetes-socat . diff --git a/contrib/mesos/pkg/minion/server.go b/contrib/mesos/pkg/minion/server.go index e7b73eeb831..e35c6602c7b 100644 --- a/contrib/mesos/pkg/minion/server.go +++ b/contrib/mesos/pkg/minion/server.go @@ -206,17 +206,23 @@ func (ms *MinionServer) launchHyperkubeServer(server string, args []string, logF } } - // use given environment, but add /usr/sbin to the path for the iptables binary used in kube-proxy + // use given environment, but add /usr/sbin and $SANDBOX/bin to the path for the iptables binary used in kube-proxy var kmEnv []string - if ms.pathOverride != "" { - env := os.Environ() - kmEnv = make([]string, 0, len(env)) - for _, e := range env { - if !strings.HasPrefix(e, "PATH=") { - kmEnv = append(kmEnv, e) + env := os.Environ() + kmEnv = make([]string, 0, len(env)) + for _, e := range env { + if !strings.HasPrefix(e, "PATH=") { + kmEnv = append(kmEnv, e) + } else { + if ms.pathOverride != "" { + e = "PATH=" + ms.pathOverride } + pwd, err := os.Getwd() + if err != nil { + log.Fatalf("Cannot get current directory: %v", err) + } + kmEnv = append(kmEnv, fmt.Sprintf("%s:%s", e, path.Join(pwd, "bin"))) } - kmEnv = append(kmEnv, "PATH="+ms.pathOverride) } t := tasks.New(server, ms.kmBinary, kmArgs, kmEnv, writerFunc) diff --git a/contrib/mesos/pkg/scheduler/service/service.go b/contrib/mesos/pkg/scheduler/service/service.go index 50713a784bd..935bec0eba5 100644 --- a/contrib/mesos/pkg/scheduler/service/service.go +++ b/contrib/mesos/pkg/scheduler/service/service.go @@ -27,6 +27,7 @@ import ( "os" "os/exec" "os/user" + "path/filepath" "strconv" "strings" "sync" @@ -150,6 +151,7 @@ type SchedulerServer struct { ContainPodResources bool AccountForPodResources bool nodeRelistPeriod time.Duration + SandboxOverlay string executable string // path to the binary running this service client *client.Client @@ -258,6 +260,7 @@ func (s *SchedulerServer) addCoreFlags(fs *pflag.FlagSet) { fs.BoolVar(&s.ExecutorBindall, "executor-bindall", s.ExecutorBindall, "When true will set -address of the executor to 0.0.0.0.") fs.DurationVar(&s.ExecutorSuicideTimeout, "executor-suicide-timeout", s.ExecutorSuicideTimeout, "Executor self-terminates after this period of inactivity. Zero disables suicide watch.") fs.DurationVar(&s.LaunchGracePeriod, "mesos-launch-grace-period", s.LaunchGracePeriod, "Launch grace period after which launching tasks will be cancelled. Zero disables launch cancellation.") + fs.StringVar(&s.SandboxOverlay, "mesos-sandbox-overlay", s.SandboxOverlay, "Path to an archive extracted in the sandbox.") fs.BoolVar(&s.ProxyBindall, "proxy-bindall", s.ProxyBindall, "When true pass -proxy-bindall to the executor.") fs.BoolVar(&s.RunProxy, "run-proxy", s.RunProxy, "Run the kube-proxy as a side process of the executor.") @@ -366,6 +369,11 @@ func (s *SchedulerServer) prepareExecutorInfo(hks hyperkube.Interface) (*mesos.E ci.Arguments = append(ci.Arguments, fmt.Sprintf("--max-log-age=%d", s.MinionLogMaxAgeInDays)) } + if s.SandboxOverlay != "" { + uri, _ := s.serveFrameworkArtifact(s.SandboxOverlay) + ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(false), Extract: proto.Bool(true)}) + } + if s.DockerCfgPath != "" { uri := s.serveFrameworkArtifactWithFilename(s.DockerCfgPath, ".dockercfg") ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(false), Extract: proto.Bool(false)}) diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index b44930ec655..c86c82f4645 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -178,10 +178,11 @@ mesos-authentication-secret-file mesos-cgroup-prefix mesos-executor-cpus mesos-executor-mem +mesos-launch-grace-period mesos-master mesos-role +mesos-sandbox-overlay mesos-user -mesos-launch-grace-period minimum-container-ttl-duration minion-max-log-age minion-max-log-backups @@ -308,4 +309,3 @@ terminated-pod-gc-threshold reconcile-cidr register-schedulable repair-malformed-updates - From 60ac1b932f5de2cdeb0d923a2280f5eb1e05b4d4 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 22 Oct 2015 11:34:00 -0700 Subject: [PATCH 2/5] Simplify serveFrameworkArtifact with library function for basenames --- contrib/mesos/pkg/scheduler/service/service.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/contrib/mesos/pkg/scheduler/service/service.go b/contrib/mesos/pkg/scheduler/service/service.go index 935bec0eba5..72955adc26c 100644 --- a/contrib/mesos/pkg/scheduler/service/service.go +++ b/contrib/mesos/pkg/scheduler/service/service.go @@ -295,15 +295,7 @@ func (s *SchedulerServer) AddHyperkubeFlags(fs *pflag.FlagSet) { // returns (downloadURI, basename(path)) func (s *SchedulerServer) serveFrameworkArtifact(path string) (string, string) { - pathSplit := strings.Split(path, "/") - - var basename string - if len(pathSplit) > 0 { - basename = pathSplit[len(pathSplit)-1] - } else { - basename = path - } - + basename := filepath.Base(path) return s.serveFrameworkArtifactWithFilename(path, basename), basename } From ed09e5c29de76926f896c4158f60049248ab9923 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Sat, 24 Oct 2015 21:40:19 -0700 Subject: [PATCH 3/5] Add supported archive format for --mesos-sandbox-overlay --- contrib/mesos/pkg/scheduler/service/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/mesos/pkg/scheduler/service/service.go b/contrib/mesos/pkg/scheduler/service/service.go index 72955adc26c..34e64daa122 100644 --- a/contrib/mesos/pkg/scheduler/service/service.go +++ b/contrib/mesos/pkg/scheduler/service/service.go @@ -260,7 +260,7 @@ func (s *SchedulerServer) addCoreFlags(fs *pflag.FlagSet) { fs.BoolVar(&s.ExecutorBindall, "executor-bindall", s.ExecutorBindall, "When true will set -address of the executor to 0.0.0.0.") fs.DurationVar(&s.ExecutorSuicideTimeout, "executor-suicide-timeout", s.ExecutorSuicideTimeout, "Executor self-terminates after this period of inactivity. Zero disables suicide watch.") fs.DurationVar(&s.LaunchGracePeriod, "mesos-launch-grace-period", s.LaunchGracePeriod, "Launch grace period after which launching tasks will be cancelled. Zero disables launch cancellation.") - fs.StringVar(&s.SandboxOverlay, "mesos-sandbox-overlay", s.SandboxOverlay, "Path to an archive extracted in the sandbox.") + fs.StringVar(&s.SandboxOverlay, "mesos-sandbox-overlay", s.SandboxOverlay, "Path to an archive (tar.gz, tar.bz2 or zip) extracted into the sandbox.") fs.BoolVar(&s.ProxyBindall, "proxy-bindall", s.ProxyBindall, "When true pass -proxy-bindall to the executor.") fs.BoolVar(&s.RunProxy, "run-proxy", s.RunProxy, "Run the kube-proxy as a side process of the executor.") From 1cebf2f014154d1f07839304605eef5c0c324fbc Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Sat, 24 Oct 2015 22:45:13 -0700 Subject: [PATCH 4/5] Add MESOS_DOCKER_OVERLAY_DIR variable to set overlay volume dir --- cluster/mesos/docker/km/build.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cluster/mesos/docker/km/build.sh b/cluster/mesos/docker/km/build.sh index cc4f38d6eb8..dd35331746c 100755 --- a/cluster/mesos/docker/km/build.sh +++ b/cluster/mesos/docker/km/build.sh @@ -48,9 +48,10 @@ kube_bin_path=$(dirname ${km_path}) common_bin_path=$(cd ${script_dir}/../common/bin && pwd -P) # download nsenter and socat -mkdir -p "${script_dir}/overlay" -docker run --rm -v "${script_dir}/overlay:/target" jpetazzo/nsenter -docker run --rm -v "${script_dir}/overlay:/target" mesosphere/kubernetes-socat +overlay_dir=${MESOS_DOCKER_OVERLAY_DIR:-${script_dir}/overlay} +mkdir -p "${overlay_dir}" +docker run --rm -v "${overlay_dir}:/target" jpetazzo/nsenter +docker run --rm -v "${overlay_dir}:/target" mesosphere/kubernetes-socat cd "${KUBE_ROOT}" @@ -61,6 +62,7 @@ echo "Workspace created: ${workspace}" cleanup() { rm -rf "${workspace}" + rm -f "${overlay_dir}/*" echo "Workspace deleted: ${workspace}" } trap 'cleanup' EXIT @@ -81,8 +83,8 @@ cp "${script_dir}/opt/"* "${workspace}/opt/" # package up the sandbox overay mkdir -p "${workspace}/overlay/bin" -cp -a "${script_dir}/overlay/nsenter" "${workspace}/overlay/bin" -cp -a "${script_dir}/overlay/socat" "${workspace}/overlay/bin" +cp -a "${overlay_dir}/nsenter" "${workspace}/overlay/bin" +cp -a "${overlay_dir}/socat" "${workspace}/overlay/bin" chmod +x "${workspace}/overlay/bin/"* cd "${workspace}/overlay" && tar -czvf "${workspace}/opt/sandbox-overlay.tar.gz" . && cd - From 8fb77ce13ef98b7cfe253fc90db4f4dc7d025743 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 3 Nov 2015 11:25:56 +0100 Subject: [PATCH 5/5] Exit scheduler if sandbox overlay does not exist --- contrib/mesos/pkg/scheduler/service/service.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/mesos/pkg/scheduler/service/service.go b/contrib/mesos/pkg/scheduler/service/service.go index 34e64daa122..262b9276872 100644 --- a/contrib/mesos/pkg/scheduler/service/service.go +++ b/contrib/mesos/pkg/scheduler/service/service.go @@ -362,6 +362,9 @@ func (s *SchedulerServer) prepareExecutorInfo(hks hyperkube.Interface) (*mesos.E } if s.SandboxOverlay != "" { + if _, err := os.Stat(s.SandboxOverlay); os.IsNotExist(err) { + log.Fatalf("Sandbox overlay archive not found: %s", s.SandboxOverlay) + } uri, _ := s.serveFrameworkArtifact(s.SandboxOverlay) ci.Uris = append(ci.Uris, &mesos.CommandInfo_URI{Value: proto.String(uri), Executable: proto.Bool(false), Extract: proto.Bool(true)}) }