From 2a0d7ebb11f70aa53548488f9af9e364c093b2df Mon Sep 17 00:00:00 2001 From: Andy Goldstein Date: Wed, 27 May 2015 12:51:01 +0000 Subject: [PATCH] Add support for pluggable Docker exec handlers Add support for pluggable Docker exec handlers. The default handler is now Docker's native exec API call. The previous default, nsenter, can be selected by passing --docker-exec-handler=nsenter when starting the kubelet. --- cmd/kubelet/app/server.go | 20 +++- pkg/kubelet/dockertools/exec.go | 141 ++++++++++++++++++++++++ pkg/kubelet/dockertools/fake_manager.go | 2 +- pkg/kubelet/dockertools/manager.go | 70 ++---------- pkg/kubelet/kubelet.go | 6 +- 5 files changed, 176 insertions(+), 63 deletions(-) create mode 100644 pkg/kubelet/dockertools/exec.go diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 0c6d75cc30d..66d1fe0e7b2 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -111,6 +111,7 @@ type KubeletServer struct { SystemContainer string ConfigureCBR0 bool MaxPods int + DockerExecHandlerName string // Flags intended for testing @@ -173,6 +174,7 @@ func NewKubeletServer() *KubeletServer { DockerDaemonContainer: "/docker-daemon", SystemContainer: "", ConfigureCBR0: false, + DockerExecHandlerName: "native", } } @@ -233,6 +235,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.SystemContainer, "system-container", s.SystemContainer, "Optional resource-only container in which to place all non-kernel processes that are not already in a container. Empty for no container. Rolling back the flag requires a reboot. (Default: \"\").") fs.BoolVar(&s.ConfigureCBR0, "configure-cbr0", s.ConfigureCBR0, "If true, kubelet will configure cbr0 based on Node.Spec.PodCIDR.") fs.IntVar(&s.MaxPods, "max-pods", 100, "Number of Pods that can run on this Kubelet.") + fs.StringVar(&s.DockerExecHandlerName, "docker-exec-handler", s.DockerExecHandlerName, "Handler to use when executing a command in a container. Valid values are 'native' and 'nsenter'. Defaults to 'native'.") // Flags intended for testing, not recommended used in production environments. fs.BoolVar(&s.ReallyCrashForTesting, "really-crash-for-testing", s.ReallyCrashForTesting, "If true, when panics occur crash. Intended for testing.") @@ -306,6 +309,17 @@ func (s *KubeletServer) Run(_ []string) error { mounter = &mount.NsenterMounter{} } + var dockerExecHandler dockertools.ExecHandler + switch s.DockerExecHandlerName { + case "native": + dockerExecHandler = &dockertools.NativeExecHandler{} + case "nsenter": + dockerExecHandler = &dockertools.NsenterExecHandler{} + default: + glog.Warningf("Unknown Docker exec handler %q; defaulting to native", s.DockerExecHandlerName) + dockerExecHandler = &dockertools.NativeExecHandler{} + } + kcfg := KubeletConfig{ Address: s.Address, AllowPrivileged: s.AllowPrivileged, @@ -352,6 +366,7 @@ func (s *KubeletServer) Run(_ []string) error { SystemContainer: s.SystemContainer, ConfigureCBR0: s.ConfigureCBR0, MaxPods: s.MaxPods, + DockerExecHandler: dockerExecHandler, } if err := RunKubelet(&kcfg, nil); err != nil { @@ -518,6 +533,7 @@ func SimpleKubelet(client *client.Client, DockerDaemonContainer: "/docker-daemon", SystemContainer: "", MaxPods: 32, + DockerExecHandler: &dockertools.NativeExecHandler{}, } return &kcfg } @@ -655,6 +671,7 @@ type KubeletConfig struct { SystemContainer string ConfigureCBR0 bool MaxPods int + DockerExecHandler dockertools.ExecHandler } func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.PodConfig, err error) { @@ -708,7 +725,8 @@ func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod kc.DockerDaemonContainer, kc.SystemContainer, kc.ConfigureCBR0, - kc.MaxPods) + kc.MaxPods, + kc.DockerExecHandler) if err != nil { return nil, nil, err diff --git a/pkg/kubelet/dockertools/exec.go b/pkg/kubelet/dockertools/exec.go new file mode 100644 index 00000000000..e0945bd22d1 --- /dev/null +++ b/pkg/kubelet/dockertools/exec.go @@ -0,0 +1,141 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockertools + +import ( + "fmt" + "io" + "os" + "os/exec" + "time" + + kubecontainer "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/container" + "github.com/fsouza/go-dockerclient" +) + +// ExecHandler knows how to execute a command in a running Docker container. +type ExecHandler interface { + ExecInContainer(client DockerInterface, container *docker.Container, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool) error +} + +// NsenterExecHandler executes commands in Docker containers using nsenter. +type NsenterExecHandler struct{} + +// TODO should we support nsenter in a container, running with elevated privs and --pid=host? +func (*NsenterExecHandler) ExecInContainer(client DockerInterface, container *docker.Container, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool) error { + nsenter, err := exec.LookPath("nsenter") + if err != nil { + return fmt.Errorf("exec unavailable - unable to locate nsenter") + } + + containerPid := container.State.Pid + + // TODO what if the container doesn't have `env`??? + args := []string{"-t", fmt.Sprintf("%d", containerPid), "-m", "-i", "-u", "-n", "-p", "--", "env", "-i"} + args = append(args, fmt.Sprintf("HOSTNAME=%s", container.Config.Hostname)) + args = append(args, container.Config.Env...) + args = append(args, cmd...) + command := exec.Command(nsenter, args...) + if tty { + p, err := kubecontainer.StartPty(command) + if err != nil { + return err + } + defer p.Close() + + // make sure to close the stdout stream + defer stdout.Close() + + if stdin != nil { + go io.Copy(p, stdin) + } + + if stdout != nil { + go io.Copy(stdout, p) + } + + return command.Wait() + } else { + if stdin != nil { + // Use an os.Pipe here as it returns true *os.File objects. + // This way, if you run 'kubectl exec -i bash' (no tty) and type 'exit', + // the call below to command.Run() can unblock because its Stdin is the read half + // of the pipe. + r, w, err := os.Pipe() + if err != nil { + return err + } + go io.Copy(w, stdin) + + command.Stdin = r + } + if stdout != nil { + command.Stdout = stdout + } + if stderr != nil { + command.Stderr = stderr + } + + return command.Run() + } +} + +// NativeExecHandler executes commands in Docker containers using Docker's exec API. +type NativeExecHandler struct{} + +func (*NativeExecHandler) ExecInContainer(client DockerInterface, container *docker.Container, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool) error { + createOpts := docker.CreateExecOptions{ + Container: container.ID, + Cmd: cmd, + AttachStdin: stdin != nil, + AttachStdout: stdout != nil, + AttachStderr: stderr != nil, + Tty: tty, + } + execObj, err := client.CreateExec(createOpts) + if err != nil { + return fmt.Errorf("failed to exec in container - Exec setup failed - %v", err) + } + startOpts := docker.StartExecOptions{ + Detach: false, + InputStream: stdin, + OutputStream: stdout, + ErrorStream: stderr, + Tty: tty, + RawTerminal: tty, + } + err = client.StartExec(execObj.ID, startOpts) + if err != nil { + return err + } + tick := time.Tick(2 * time.Second) + for { + inspect, err2 := client.InspectExec(execObj.ID) + if err2 != nil { + return err2 + } + if !inspect.Running { + if inspect.ExitCode != 0 { + err = &dockerExitError{inspect} + } + break + } + <-tick + } + + return err +} diff --git a/pkg/kubelet/dockertools/fake_manager.go b/pkg/kubelet/dockertools/fake_manager.go index 1221b933acd..287dc771a1b 100644 --- a/pkg/kubelet/dockertools/fake_manager.go +++ b/pkg/kubelet/dockertools/fake_manager.go @@ -40,7 +40,7 @@ func NewFakeDockerManager( runtimeHooks kubecontainer.RuntimeHooks) *DockerManager { dm := NewDockerManager(client, recorder, readinessManager, containerRefManager, podInfraContainerImage, qps, - burst, containerLogsDir, osInterface, networkPlugin, generator, httpClient, runtimeHooks) + burst, containerLogsDir, osInterface, networkPlugin, generator, httpClient, runtimeHooks, &NativeExecHandler{}) dm.Puller = &FakeDockerPuller{} dm.prober = prober.New(nil, readinessManager, containerRefManager, recorder) return dm diff --git a/pkg/kubelet/dockertools/manager.go b/pkg/kubelet/dockertools/manager.go index d1bbf459493..dce256a6043 100644 --- a/pkg/kubelet/dockertools/manager.go +++ b/pkg/kubelet/dockertools/manager.go @@ -106,6 +106,9 @@ type DockerManager struct { // Hooks injected into the container runtime. runtimeHooks kubecontainer.RuntimeHooks + + // Handler used to execute commands in containers. + execHandler ExecHandler } func NewDockerManager( @@ -121,7 +124,8 @@ func NewDockerManager( networkPlugin network.NetworkPlugin, generator kubecontainer.RunContainerOptionsGenerator, httpClient kubeletTypes.HttpGetter, - runtimeHooks kubecontainer.RuntimeHooks) *DockerManager { + runtimeHooks kubecontainer.RuntimeHooks, + execHandler ExecHandler) *DockerManager { // Work out the location of the Docker runtime, defaulting to /var/lib/docker // if there are any problems. dockerRoot := "/var/lib/docker" @@ -153,6 +157,7 @@ func NewDockerManager( } reasonCache := stringCache{cache: lru.New(maxReasonCacheEntries)} + dm := &DockerManager{ client: client, recorder: recorder, @@ -168,6 +173,7 @@ func NewDockerManager( prober: nil, generator: generator, runtimeHooks: runtimeHooks, + execHandler: execHandler, } dm.runner = lifecycle.NewHandlerRunner(httpClient, dm, dm) dm.prober = prober.New(dm, readinessManager, containerRefManager, recorder) @@ -969,78 +975,24 @@ func (d *dockerExitError) ExitStatus() int { return d.Inspect.ExitCode } -// ExecInContainer uses nsenter to run the command inside the container identified by containerID. +// ExecInContainer runs the command inside the container identified by containerID. // // TODO: -// - match cgroups of container -// - should we support `docker exec`? -// - should we support nsenter in a container, running with elevated privs and --pid=host? // - use strong type for containerId func (dm *DockerManager) ExecInContainer(containerId string, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool) error { - nsenter, err := exec.LookPath("nsenter") - if err != nil { - return fmt.Errorf("exec unavailable - unable to locate nsenter") + if dm.execHandler == nil { + return errors.New("unable to exec without an exec handler") } container, err := dm.client.InspectContainer(containerId) if err != nil { return err } - if !container.State.Running { return fmt.Errorf("container not running (%s)", container) } - containerPid := container.State.Pid - - // TODO what if the container doesn't have `env`??? - args := []string{"-t", fmt.Sprintf("%d", containerPid), "-m", "-i", "-u", "-n", "-p", "--", "env", "-i"} - args = append(args, fmt.Sprintf("HOSTNAME=%s", container.Config.Hostname)) - args = append(args, container.Config.Env...) - args = append(args, cmd...) - command := exec.Command(nsenter, args...) - if tty { - p, err := kubecontainer.StartPty(command) - if err != nil { - return err - } - defer p.Close() - - // make sure to close the stdout stream - defer stdout.Close() - - if stdin != nil { - go io.Copy(p, stdin) - } - - if stdout != nil { - go io.Copy(stdout, p) - } - - return command.Wait() - } else { - if stdin != nil { - // Use an os.Pipe here as it returns true *os.File objects. - // This way, if you run 'kubectl exec -i bash' (no tty) and type 'exit', - // the call below to command.Run() can unblock because its Stdin is the read half - // of the pipe. - r, w, err := os.Pipe() - if err != nil { - return err - } - go io.Copy(w, stdin) - - command.Stdin = r - } - if stdout != nil { - command.Stdout = stdout - } - if stderr != nil { - command.Stderr = stderr - } - - return command.Run() - } + return dm.execHandler.ExecInContainer(dm.client, container, cmd, stdin, stdout, stderr, tty) } // PortForward executes socat in the pod's network namespace and copies diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 2b218c190da..c1064fb1647 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -141,7 +141,8 @@ func NewMainKubelet( dockerDaemonContainer string, systemContainer string, configureCBR0 bool, - pods int) (*Kubelet, error) { + pods int, + dockerExecHandler dockertools.ExecHandler) (*Kubelet, error) { if rootDirectory == "" { return nil, fmt.Errorf("invalid root directory %q", rootDirectory) } @@ -278,7 +279,8 @@ func NewMainKubelet( klet.networkPlugin, klet, klet.httpClient, - newKubeletRuntimeHooks(recorder)) + newKubeletRuntimeHooks(recorder), + dockerExecHandler) case "rkt": conf := &rkt.Config{InsecureSkipVerify: true} rktRuntime, err := rkt.New(