From 7fbd290167a898be93bc42e2ee868e6e266d1201 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sun, 30 Aug 2015 19:16:40 +0000 Subject: [PATCH] moved cgroup-root detection to minion service; kube proxy now configured to run in mesos container --- contrib/mesos/pkg/executor/config/config.go | 1 - contrib/mesos/pkg/executor/service/service.go | 36 +----------------- contrib/mesos/pkg/minion/config/config.go | 2 + contrib/mesos/pkg/minion/server.go | 38 +++++++++++++++++++ .../mesos/pkg/scheduler/service/service.go | 8 ++-- hack/verify-flags/known-flags.txt | 3 +- 6 files changed, 46 insertions(+), 42 deletions(-) diff --git a/contrib/mesos/pkg/executor/config/config.go b/contrib/mesos/pkg/executor/config/config.go index 22fc378a6d9..999058dbc8b 100644 --- a/contrib/mesos/pkg/executor/config/config.go +++ b/contrib/mesos/pkg/executor/config/config.go @@ -26,5 +26,4 @@ const ( DefaultInfoSource = "kubernetes" DefaultInfoName = "Kubelet-Executor" DefaultSuicideTimeout = 20 * time.Minute - DefaultCgroupPrefix = "mesos" ) diff --git a/contrib/mesos/pkg/executor/service/service.go b/contrib/mesos/pkg/executor/service/service.go index a026ba7314f..777d23e1857 100644 --- a/contrib/mesos/pkg/executor/service/service.go +++ b/contrib/mesos/pkg/executor/service/service.go @@ -17,13 +17,11 @@ limitations under the License. package service import ( - "fmt" "io" "math/rand" "net" "net/http" "os" - "path" "path/filepath" "strconv" "strings" @@ -63,33 +61,12 @@ type KubeletExecutorServer struct { SuicideTimeout time.Duration ShutdownFD int ShutdownFIFO string - cgroupRoot string - cgroupPrefix string -} - -func findMesosCgroup(prefix string) string { - // derive our cgroup from MESOS_DIRECTORY environment - mesosDir := os.Getenv("MESOS_DIRECTORY") - if mesosDir == "" { - log.V(2).Infof("cannot derive executor's cgroup because MESOS_DIRECTORY is empty") - return "" - } - - containerId := path.Base(mesosDir) - if containerId == "" { - log.V(2).Infof("cannot derive executor's cgroup from MESOS_DIRECTORY=%q", mesosDir) - return "" - } - trimmedPrefix := strings.Trim(prefix, "/") - cgroupRoot := fmt.Sprintf("/%s/%v", trimmedPrefix, containerId) - return cgroupRoot } func NewKubeletExecutorServer() *KubeletExecutorServer { k := &KubeletExecutorServer{ KubeletServer: app.NewKubeletServer(), SuicideTimeout: config.DefaultSuicideTimeout, - cgroupPrefix: config.DefaultCgroupPrefix, } if pwd, err := os.Getwd(); err != nil { log.Warningf("failed to determine current directory: %v", err) @@ -107,7 +84,6 @@ func (s *KubeletExecutorServer) AddFlags(fs *pflag.FlagSet) { fs.DurationVar(&s.SuicideTimeout, "suicide-timeout", s.SuicideTimeout, "Self-terminate after this period of inactivity. Zero disables suicide watch.") fs.IntVar(&s.ShutdownFD, "shutdown-fd", s.ShutdownFD, "File descriptor used to signal shutdown to external watchers, requires shutdown-fifo flag") fs.StringVar(&s.ShutdownFIFO, "shutdown-fifo", s.ShutdownFIFO, "FIFO used to signal shutdown to external watchers, requires shutdown-fd flag") - fs.StringVar(&s.cgroupPrefix, "cgroup-prefix", s.cgroupPrefix, "The cgroup prefix concatenated with MESOS_DIRECTORY must give the executor cgroup set by Mesos") } // returns a Closer that should be closed to signal impending shutdown, but only if ShutdownFD @@ -131,21 +107,11 @@ func (s *KubeletExecutorServer) Run(hks hyperkube.Interface, _ []string) error { log.Info(err) } - // derive the executor cgroup and use it as docker container cgroup root - mesosCgroup := findMesosCgroup(s.cgroupPrefix) - s.cgroupRoot = mesosCgroup - log.V(2).Infof("passing cgroup %q to the kubelet as cgroup root", s.CgroupRoot) - // empty string for the docker and system containers (= cgroup paths). This // stops the kubelet taking any control over other system processes. s.SystemContainer = "" s.DockerDaemonContainer = "" - // We set kubelet container to its own cgroup below the executor cgroup. - // In contrast to the docker and system container, this has no other - // undesired side-effects. - s.ResourceContainer = mesosCgroup + "/kubelet" - // create apiserver client var apiclient *client.Client clientConfig, err := s.CreateAPIServerClientConfig() @@ -253,7 +219,7 @@ func (s *KubeletExecutorServer) Run(hks hyperkube.Interface, _ []string) error { Cloud: nil, // TODO(jdef) Cloud, specifying null here because we don't want all kubelets polling mesos-master; need to account for this in the cloudprovider impl NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency, ResourceContainer: s.ResourceContainer, - CgroupRoot: s.cgroupRoot, + CgroupRoot: s.CgroupRoot, ContainerRuntime: s.ContainerRuntime, Mounter: mounter, DockerDaemonContainer: s.DockerDaemonContainer, diff --git a/contrib/mesos/pkg/minion/config/config.go b/contrib/mesos/pkg/minion/config/config.go index af5a2ce94ef..58cd5d99465 100644 --- a/contrib/mesos/pkg/minion/config/config.go +++ b/contrib/mesos/pkg/minion/config/config.go @@ -23,6 +23,8 @@ import ( const ( DefaultLogMaxBackups = 5 // how many backup to keep DefaultLogMaxAgeInDays = 7 // after how many days to rotate at most + + DefaultCgroupPrefix = "mesos" ) // DefaultLogMaxSize returns the maximal log file size before rotation diff --git a/contrib/mesos/pkg/minion/server.go b/contrib/mesos/pkg/minion/server.go index 1588fb10847..7a6adff4cc1 100644 --- a/contrib/mesos/pkg/minion/server.go +++ b/contrib/mesos/pkg/minion/server.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "os" "os/exec" + "path" "strings" "time" @@ -51,6 +52,8 @@ type MinionServer struct { exit chan error // to signal fatal errors pathOverride string // the PATH environment for the sub-processes + cgroupPrefix string // e.g. mesos + cgroupRoot string // e.g. /mesos/{container-id}, determined at runtime logMaxSize resource.Quantity logMaxBackups int @@ -69,6 +72,7 @@ func NewMinionServer() *MinionServer { done: make(chan struct{}), exit: make(chan error), + cgroupPrefix: config.DefaultCgroupPrefix, logMaxSize: config.DefaultLogMaxSize(), logMaxBackups: config.DefaultLogMaxBackups, logMaxAgeInDays: config.DefaultLogMaxAgeInDays, @@ -102,6 +106,22 @@ func filterArgsByFlagSet(args []string, flags *pflag.FlagSet) ([]string, []strin return matched, notMatched } +func findMesosCgroup(prefix string) string { + // derive our cgroup from MESOS_DIRECTORY environment + mesosDir := os.Getenv("MESOS_DIRECTORY") + if mesosDir == "" { + log.V(2).Infof("cannot derive executor's cgroup because MESOS_DIRECTORY is empty") + return "" + } + + containerId := path.Base(mesosDir) + if containerId == "" { + log.V(2).Infof("cannot derive executor's cgroup from MESOS_DIRECTORY=%q", mesosDir) + return "" + } + return path.Join("/", prefix, containerId) +} + func (ms *MinionServer) launchProxyServer() { bindAddress := "0.0.0.0" if !ms.proxyBindall { @@ -111,6 +131,7 @@ func (ms *MinionServer) launchProxyServer() { fmt.Sprintf("--bind-address=%s", bindAddress), fmt.Sprintf("--v=%d", ms.proxyLogV), "--logtostderr=true", + "--resource-container=" + path.Join("/", ms.cgroupRoot, "kube-proxy"), } if ms.clientConfig.Host != "" { @@ -132,6 +153,11 @@ func (ms *MinionServer) launchExecutorServer() { ms.AddExecutorFlags(executorFlags) executorArgs, _ := filterArgsByFlagSet(allArgs, executorFlags) + executorArgs = append(executorArgs, "--resource-container="+path.Join("/", ms.cgroupRoot, "kubelet")) + if ms.cgroupRoot != "" { + executorArgs = append(executorArgs, "--cgroup-root="+ms.cgroupRoot) + } + // run executor and quit minion server when this exits cleanly err := ms.launchHyperkubeServer(hyperkube.CommandExecutor, &executorArgs, "executor.log") if err != nil { @@ -257,6 +283,17 @@ func (ms *MinionServer) Run(hks hyperkube.Interface, _ []string) error { } ms.clientConfig = clientConfig + // derive the executor cgroup and use it as: + // - pod container cgroup root (e.g. docker cgroup-parent) + // - parent of kubelet container + // - parent of kube-proxy container + ms.cgroupRoot = findMesosCgroup(ms.cgroupPrefix) + cgroupLogger := log.Infof + if ms.cgroupRoot == "" { + cgroupLogger = log.Warningf + } + cgroupLogger("using cgroup-root %q", ms.cgroupRoot) + // run subprocesses until ms.done is closed on return of this function defer close(ms.done) if ms.runProxy { @@ -275,6 +312,7 @@ func (ms *MinionServer) AddExecutorFlags(fs *pflag.FlagSet) { func (ms *MinionServer) AddMinionFlags(fs *pflag.FlagSet) { // general minion flags + fs.StringVar(&ms.cgroupPrefix, "mesos-cgroup-prefix", ms.cgroupPrefix, "The cgroup prefix concatenated with MESOS_DIRECTORY must give the executor cgroup set by Mesos") fs.BoolVar(&ms.privateMountNS, "private-mountns", ms.privateMountNS, "Enter a private mount NS before spawning procs (linux only). Experimental, not yet compatible with k8s volumes.") fs.StringVar(&ms.pathOverride, "path-override", ms.pathOverride, "Override the PATH in the environment of the sub-processes.") diff --git a/contrib/mesos/pkg/scheduler/service/service.go b/contrib/mesos/pkg/scheduler/service/service.go index c5df03df1cb..541fe47a159 100644 --- a/contrib/mesos/pkg/scheduler/service/service.go +++ b/contrib/mesos/pkg/scheduler/service/service.go @@ -95,13 +95,13 @@ type SchedulerServer struct { MesosRole string MesosAuthPrincipal string MesosAuthSecretFile string + MesosCgroupPrefix string Checkpoint bool FailoverTimeout float64 ExecutorLogV int ExecutorBindall bool ExecutorSuicideTimeout time.Duration - ExecutorCgroupPrefix string RunProxy bool ProxyBindall bool @@ -163,7 +163,6 @@ func NewSchedulerServer() *SchedulerServer { RunProxy: true, ExecutorSuicideTimeout: execcfg.DefaultSuicideTimeout, - ExecutorCgroupPrefix: execcfg.DefaultCgroupPrefix, DefaultContainerCPULimit: mresource.DefaultDefaultContainerCPULimit, DefaultContainerMemLimit: mresource.DefaultDefaultContainerMemLimit, @@ -172,6 +171,7 @@ func NewSchedulerServer() *SchedulerServer { MinionLogMaxAgeInDays: minioncfg.DefaultLogMaxAgeInDays, MesosAuthProvider: sasl.ProviderName, + MesosCgroupPrefix: minioncfg.DefaultCgroupPrefix, MesosMaster: defaultMesosMaster, MesosUser: defaultMesosUser, ReconcileInterval: defaultReconcileInterval, @@ -215,6 +215,7 @@ func (s *SchedulerServer) addCoreFlags(fs *pflag.FlagSet) { fs.StringVar(&s.MesosAuthSecretFile, "mesos-authentication-secret-file", s.MesosAuthSecretFile, "Mesos authentication secret file.") fs.StringVar(&s.MesosAuthProvider, "mesos-authentication-provider", s.MesosAuthProvider, fmt.Sprintf("Authentication provider to use, default is SASL that supports mechanisms: %+v", mech.ListSupported())) fs.StringVar(&s.DockerCfgPath, "dockercfg-path", s.DockerCfgPath, "Path to a dockercfg file that will be used by the docker instance of the minions.") + fs.StringVar(&s.MesosCgroupPrefix, "mesos-cgroup-prefix", s.MesosCgroupPrefix, "The cgroup prefix concatenated with MESOS_DIRECTORY must give the executor cgroup set by Mesos") fs.BoolVar(&s.Checkpoint, "checkpoint", s.Checkpoint, "Enable/disable checkpointing for the kubernetes-mesos framework.") fs.Float64Var(&s.FailoverTimeout, "failover-timeout", s.FailoverTimeout, fmt.Sprintf("Framework failover timeout, in sec.")) fs.UintVar(&s.DriverPort, "driver-port", s.DriverPort, "Port that the Mesos scheduler driver process should listen on.") @@ -234,7 +235,6 @@ func (s *SchedulerServer) addCoreFlags(fs *pflag.FlagSet) { fs.IntVar(&s.ExecutorLogV, "executor-logv", s.ExecutorLogV, "Logging verbosity of spawned minion and executor processes.") fs.BoolVar(&s.ExecutorBindall, "executor-bindall", s.ExecutorBindall, "When true will set -address of the executor to 0.0.0.0.") fs.DurationVar(&s.ExecutorSuicideTimeout, "executor-suicide-timeout", s.ExecutorSuicideTimeout, "Executor self-terminates after this period of inactivity. Zero disables suicide watch.") - fs.StringVar(&s.ExecutorCgroupPrefix, "executor-cgroup-prefix", s.ExecutorCgroupPrefix, "The cgroup prefix concatenated with MESOS_DIRECTORY must give the executor cgroup set by Mesos") fs.BoolVar(&s.ProxyBindall, "proxy-bindall", s.ProxyBindall, "When true pass -proxy-bindall to the executor.") fs.BoolVar(&s.RunProxy, "run-proxy", s.RunProxy, "Run the kube-proxy as a side process of the executor.") @@ -364,7 +364,7 @@ func (s *SchedulerServer) prepareExecutorInfo(hks hyperkube.Interface) (*mesos.E ci.Arguments = append(ci.Arguments, "--address=0.0.0.0") } - ci.Arguments = append(ci.Arguments, fmt.Sprintf("--cgroup-prefix=%v", s.ExecutorCgroupPrefix)) + ci.Arguments = append(ci.Arguments, fmt.Sprintf("--mesos-cgroup-prefix=%v", s.MesosCgroupPrefix)) ci.Arguments = append(ci.Arguments, fmt.Sprintf("--cadvisor-port=%v", s.KubeletCadvisorPort)) ci.Arguments = append(ci.Arguments, fmt.Sprintf("--sync-frequency=%v", s.KubeletSyncFrequency)) diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index a3a4be222d8..61d5bddba52 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -28,7 +28,6 @@ bind-pods-qps cadvisor-port cert-dir certificate-authority -cgroup-prefix cgroup-root chaos-chance cleanup-iptables @@ -74,7 +73,6 @@ etcd-server etcd-servers event-ttl executor-bindall -executor-cgroup-prefix executor-logv executor-path executor-suicide-timeout @@ -152,6 +150,7 @@ max-requests-inflight mesos-authentication-principal mesos-authentication-provider mesos-authentication-secret-file +mesos-cgroup-prefix mesos-master mesos-role mesos-user