Merge pull request #45199 from yujuhong/clean-up-dockertools

Automatic merge from submit-queue

Clean up code in dockertools

Move functions and sub packages to dockershim.
Part of #43234.

cmd/kubelet/app/server.go

@@ -939,18 +939,6 @@ func RunDockershim(c *componentconfig.KubeletConfiguration, dockershimRootDir st
 	dockerClient := dockertools.ConnectToDockerOrDie(c.DockerEndpoint, c.RuntimeRequestTimeout.Duration,
 		c.ImagePullProgressDeadline.Duration)
 
-	// Initialize docker exec handler.
-	var dockerExecHandler dockertools.ExecHandler
-	switch c.DockerExecHandlerName {
-	case "native":
-		dockerExecHandler = &dockertools.NativeExecHandler{}
-	case "nsenter":
-		dockerExecHandler = &dockertools.NsenterExecHandler{}
-	default:
-		glog.Warningf("Unknown Docker exec handler %q; defaulting to native", c.DockerExecHandlerName)
-		dockerExecHandler = &dockertools.NativeExecHandler{}
-	}
-
 	// Initialize network plugin settings.
 	binDir := c.CNIBinDir
 	if binDir == "" {
@@ -976,7 +964,7 @@ func RunDockershim(c *componentconfig.KubeletConfiguration, dockershimRootDir st
 	}
 
 	ds, err := dockershim.NewDockerService(dockerClient, c.SeccompProfileRoot, c.PodInfraContainerImage,
-		streamingConfig, &pluginSettings, c.RuntimeCgroups, c.CgroupDriver, dockerExecHandler, dockershimRootDir,
+		streamingConfig, &pluginSettings, c.RuntimeCgroups, c.CgroupDriver, c.DockerExecHandlerName, dockershimRootDir,
 		!c.DockerEnableSharedPID)
 	if err != nil {
 		return err

pkg/kubelet/dockershim/BUILD

@@ -21,6 +21,7 @@ go_library(
         "docker_sandbox.go",
         "docker_service.go",
         "docker_streaming.go",
+        "exec.go",
         "helpers.go",
         "naming.go",
         "security_context.go",
@@ -37,8 +38,8 @@ go_library(
         "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/dockershim/cm:go_default_library",
         "//pkg/kubelet/dockershim/errors:go_default_library",
+        "//pkg/kubelet/dockershim/securitycontext:go_default_library",
         "//pkg/kubelet/dockertools:go_default_library",
-        "//pkg/kubelet/dockertools/securitycontext:go_default_library",
         "//pkg/kubelet/leaky:go_default_library",
         "//pkg/kubelet/network:go_default_library",
         "//pkg/kubelet/network/cni:go_default_library",
@@ -49,7 +50,10 @@ go_library(
         "//pkg/kubelet/types:go_default_library",
         "//pkg/kubelet/util/cache:go_default_library",
         "//pkg/kubelet/util/ioutils:go_default_library",
+        "//pkg/security/apparmor:go_default_library",
+        "//pkg/util/exec:go_default_library",
         "//pkg/util/hash:go_default_library",
+        "//pkg/util/term:go_default_library",
         "//vendor/github.com/blang/semver:go_default_library",
         "//vendor/github.com/docker/engine-api/types:go_default_library",
         "//vendor/github.com/docker/engine-api/types/container:go_default_library",
@@ -57,6 +61,7 @@ go_library(
         "//vendor/github.com/docker/engine-api/types/strslice:go_default_library",
         "//vendor/github.com/docker/go-connections/nat:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
@@ -86,9 +91,9 @@ go_test(
         "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/container/testing:go_default_library",
         "//pkg/kubelet/dockershim/errors:go_default_library",
+        "//pkg/kubelet/dockershim/securitycontext:go_default_library",
         "//pkg/kubelet/dockershim/testing:go_default_library",
         "//pkg/kubelet/dockertools:go_default_library",
-        "//pkg/kubelet/dockertools/securitycontext:go_default_library",
         "//pkg/kubelet/network:go_default_library",
         "//pkg/kubelet/network/testing:go_default_library",
         "//pkg/kubelet/types:go_default_library",
@@ -119,6 +124,7 @@ filegroup(
         "//pkg/kubelet/dockershim/cm:all-srcs",
         "//pkg/kubelet/dockershim/errors:all-srcs",
         "//pkg/kubelet/dockershim/remote:all-srcs",
+        "//pkg/kubelet/dockershim/securitycontext:all-srcs",
         "//pkg/kubelet/dockershim/testing:all-srcs",
     ],
     tags = ["automanaged"],

pkg/kubelet/dockershim/docker_image.go

@@ -17,6 +17,8 @@ limitations under the License.
 package dockershim
 
 import (
+	"fmt"
+
 	dockertypes "github.com/docker/engine-api/types"
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
 	"k8s.io/kubernetes/pkg/kubelet/dockertools"
@@ -80,7 +82,7 @@ func (ds *dockerService) PullImage(image *runtimeapi.ImageSpec, auth *runtimeapi
 		return "", err
 	}
 
-	return dockertools.GetImageRef(ds.client, image.Image)
+	return getImageRef(ds.client, image.Image)
 }
 
 // RemoveImage removes the image.
@@ -101,3 +103,21 @@ func (ds *dockerService) RemoveImage(image *runtimeapi.ImageSpec) error {
 	_, err = ds.client.RemoveImage(image.Image, dockertypes.ImageRemoveOptions{PruneChildren: true})
 	return err
 }
+
+// getImageRef returns the image digest if exists, or else returns the image ID.
+func getImageRef(client dockertools.DockerInterface, image string) (string, error) {
+	img, err := client.InspectImageByRef(image)
+	if err != nil {
+		return "", err
+	}
+	if img == nil {
+		return "", fmt.Errorf("unable to inspect image %s", image)
+	}
+
+	// Returns the digest if it exist.
+	if len(img.RepoDigests) > 0 {
+		return img.RepoDigests[0], nil
+	}
+
+	return img.ID, nil
+}

pkg/kubelet/dockershim/docker_service.go

@@ -20,12 +20,14 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strconv"
 	"time"
 
 	"github.com/blang/semver"
 	dockertypes "github.com/docker/engine-api/types"
 	"github.com/golang/glog"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/pkg/api/v1"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	internalapi "k8s.io/kubernetes/pkg/kubelet/api"
@@ -147,12 +149,23 @@ var internalLabelKeys []string = []string{containerTypeLabelKey, containerLogPat
 
 // NOTE: Anything passed to DockerService should be eventually handled in another way when we switch to running the shim as a different process.
 func NewDockerService(client dockertools.DockerInterface, seccompProfileRoot string, podSandboxImage string, streamingConfig *streaming.Config,
-	pluginSettings *NetworkPluginSettings, cgroupsName string, kubeCgroupDriver string, execHandler dockertools.ExecHandler, dockershimRootDir string, disableSharedPID bool) (DockerService, error) {
+	pluginSettings *NetworkPluginSettings, cgroupsName string, kubeCgroupDriver string, execHandlerName, dockershimRootDir string, disableSharedPID bool) (DockerService, error) {
 	c := dockertools.NewInstrumentedDockerInterface(client)
 	checkpointHandler, err := NewPersistentCheckpointHandler(dockershimRootDir)
 	if err != nil {
 		return nil, err
 	}
+	var execHandler ExecHandler
+	switch execHandlerName {
+	case "native":
+		execHandler = &NativeExecHandler{}
+	case "nsenter":
+		execHandler = &NsenterExecHandler{}
+	default:
+		glog.Warningf("Unknown Docker exec handler %q; defaulting to native", execHandlerName)
+		execHandler = &NativeExecHandler{}
+	}
+
 	ds := &dockerService{
 		seccompProfileRoot: seccompProfileRoot,
 		client:             c,
@@ -478,7 +491,32 @@ func (d *dockerLegacyService) GetContainerLogs(pod *v1.Pod, containerID kubecont
 	if err != nil {
 		return err
 	}
-	return dockertools.GetContainerLogs(d.client, pod, containerID, logOptions, stdout, stderr, container.Config.Tty)
+
+	var since int64
+	if logOptions.SinceSeconds != nil {
+		t := metav1.Now().Add(-time.Duration(*logOptions.SinceSeconds) * time.Second)
+		since = t.Unix()
+	}
+	if logOptions.SinceTime != nil {
+		since = logOptions.SinceTime.Unix()
+	}
+	opts := dockertypes.ContainerLogsOptions{
+		ShowStdout: true,
+		ShowStderr: true,
+		Since:      strconv.FormatInt(since, 10),
+		Timestamps: logOptions.Timestamps,
+		Follow:     logOptions.Follow,
+	}
+	if logOptions.TailLines != nil {
+		opts.Tail = strconv.FormatInt(*logOptions.TailLines, 10)
+	}
+
+	sopts := dockertools.StreamOptions{
+		OutputStream: stdout,
+		ErrorStream:  stderr,
+		RawTerminal:  container.Config.Tty,
+	}
+	return d.client.Logs(containerID.ID, opts, sopts)
 }
 
 // criSupportedLogDrivers are log drivers supported by native CRI integration.

pkg/kubelet/dockershim/docker_streaming.go

@@ -21,11 +21,15 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"os/exec"
+	"strings"
 	"time"
 
 	dockertypes "github.com/docker/engine-api/types"
+	"github.com/golang/glog"
 	"k8s.io/kubernetes/pkg/client/unversioned/remotecommand"
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/dockertools"
 	"k8s.io/kubernetes/pkg/kubelet/server/streaming"
 	"k8s.io/kubernetes/pkg/kubelet/util/ioutils"
@@ -33,7 +37,7 @@ import (
 
 type streamingRuntime struct {
 	client      dockertools.DockerInterface
-	execHandler dockertools.ExecHandler
+	execHandler ExecHandler
 }
 
 var _ streaming.Runtime = &streamingRuntime{}
@@ -57,14 +61,14 @@ func (r *streamingRuntime) Attach(containerID string, in io.Reader, out, errw io
 		return err
 	}
 
-	return dockertools.AttachContainer(r.client, containerID, in, out, errw, tty, resize)
+	return attachContainer(r.client, containerID, in, out, errw, tty, resize)
 }
 
 func (r *streamingRuntime) PortForward(podSandboxID string, port int32, stream io.ReadWriteCloser) error {
 	if port < 0 || port > math.MaxUint16 {
 		return fmt.Errorf("invalid port %d", port)
 	}
-	return dockertools.PortForward(r.client, podSandboxID, port, stream)
+	return portForward(r.client, podSandboxID, port, stream)
 }
 
 // ExecSync executes a command in the container, and returns the stdout output.
@@ -128,3 +132,83 @@ func checkContainerStatus(client dockertools.DockerInterface, containerID string
 	}
 	return container, nil
 }
+
+func attachContainer(client dockertools.DockerInterface, containerID string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error {
+	// Have to start this before the call to client.AttachToContainer because client.AttachToContainer is a blocking
+	// call :-( Otherwise, resize events don't get processed and the terminal never resizes.
+	kubecontainer.HandleResizing(resize, func(size remotecommand.TerminalSize) {
+		client.ResizeContainerTTY(containerID, int(size.Height), int(size.Width))
+	})
+
+	// TODO(random-liu): Do we really use the *Logs* field here?
+	opts := dockertypes.ContainerAttachOptions{
+		Stream: true,
+		Stdin:  stdin != nil,
+		Stdout: stdout != nil,
+		Stderr: stderr != nil,
+	}
+	sopts := dockertools.StreamOptions{
+		InputStream:  stdin,
+		OutputStream: stdout,
+		ErrorStream:  stderr,
+		RawTerminal:  tty,
+	}
+	return client.AttachToContainer(containerID, opts, sopts)
+}
+
+func portForward(client dockertools.DockerInterface, podInfraContainerID string, port int32, stream io.ReadWriteCloser) error {
+	container, err := client.InspectContainer(podInfraContainerID)
+	if err != nil {
+		return err
+	}
+
+	if !container.State.Running {
+		return fmt.Errorf("container not running (%s)", container.ID)
+	}
+
+	containerPid := container.State.Pid
+	socatPath, lookupErr := exec.LookPath("socat")
+	if lookupErr != nil {
+		return fmt.Errorf("unable to do port forwarding: socat not found.")
+	}
+
+	args := []string{"-t", fmt.Sprintf("%d", containerPid), "-n", socatPath, "-", fmt.Sprintf("TCP4:localhost:%d", port)}
+
+	nsenterPath, lookupErr := exec.LookPath("nsenter")
+	if lookupErr != nil {
+		return fmt.Errorf("unable to do port forwarding: nsenter not found.")
+	}
+
+	commandString := fmt.Sprintf("%s %s", nsenterPath, strings.Join(args, " "))
+	glog.V(4).Infof("executing port forwarding command: %s", commandString)
+
+	command := exec.Command(nsenterPath, args...)
+	command.Stdout = stream
+
+	stderr := new(bytes.Buffer)
+	command.Stderr = stderr
+
+	// If we use Stdin, command.Run() won't return until the goroutine that's copying
+	// from stream finishes. Unfortunately, if you have a client like telnet connected
+	// via port forwarding, as long as the user's telnet client is connected to the user's
+	// local listener that port forwarding sets up, the telnet session never exits. This
+	// means that even if socat has finished running, command.Run() won't ever return
+	// (because the client still has the connection and stream open).
+	//
+	// The work around is to use StdinPipe(), as Wait() (called by Run()) closes the pipe
+	// when the command (socat) exits.
+	inPipe, err := command.StdinPipe()
+	if err != nil {
+		return fmt.Errorf("unable to do port forwarding: error creating stdin pipe: %v", err)
+	}
+	go func() {
+		io.Copy(inPipe, stream)
+		inPipe.Close()
+	}()
+
+	if err := command.Run(); err != nil {
+		return fmt.Errorf("%v: %s", err, stderr.String())
+	}
+
+	return nil
+}

pkg/kubelet/dockershim/exec.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package dockertools
+package dockershim
 
 import (
 	"fmt"
@@ -25,22 +25,44 @@ import (
 
 	dockertypes "github.com/docker/engine-api/types"
 	"github.com/golang/glog"
+
 	"k8s.io/kubernetes/pkg/client/unversioned/remotecommand"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/dockertools"
 	utilexec "k8s.io/kubernetes/pkg/util/exec"
 	"k8s.io/kubernetes/pkg/util/term"
 )
 
 // ExecHandler knows how to execute a command in a running Docker container.
 type ExecHandler interface {
-	ExecInContainer(client DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error
+	ExecInContainer(client dockertools.DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error
 }
 
 // NsenterExecHandler executes commands in Docker containers using nsenter.
 type NsenterExecHandler struct{}
 
+type dockerExitError struct {
+	Inspect *dockertypes.ContainerExecInspect
+}
+
+func (d *dockerExitError) String() string {
+	return d.Error()
+}
+
+func (d *dockerExitError) Error() string {
+	return fmt.Sprintf("Error executing in Docker Container: %d", d.Inspect.ExitCode)
+}
+
+func (d *dockerExitError) Exited() bool {
+	return !d.Inspect.Running
+}
+
+func (d *dockerExitError) ExitStatus() int {
+	return d.Inspect.ExitCode
+}
+
 // TODO should we support nsenter in a container, running with elevated privs and --pid=host?
-func (*NsenterExecHandler) ExecInContainer(client DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error {
+func (*NsenterExecHandler) ExecInContainer(client dockertools.DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error {
 	nsenter, err := exec.LookPath("nsenter")
 	if err != nil {
 		return fmt.Errorf("exec unavailable - unable to locate nsenter")
@@ -111,7 +133,7 @@ func (*NsenterExecHandler) ExecInContainer(client DockerInterface, container *do
 // NativeExecHandler executes commands in Docker containers using Docker's exec API.
 type NativeExecHandler struct{}
 
-func (*NativeExecHandler) ExecInContainer(client DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error {
+func (*NativeExecHandler) ExecInContainer(client dockertools.DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error {
 	createOpts := dockertypes.ExecConfig{
 		Cmd:          cmd,
 		AttachStdin:  stdin != nil,
@@ -131,7 +153,7 @@ func (*NativeExecHandler) ExecInContainer(client DockerInterface, container *doc
 	})
 
 	startOpts := dockertypes.ExecStartCheck{Detach: false, Tty: tty}
-	streamOpts := StreamOptions{
+	streamOpts := dockertools.StreamOptions{
 		InputStream:  stdin,
 		OutputStream: stdout,
 		ErrorStream:  stderr,

pkg/kubelet/dockershim/helpers.go

@@ -17,7 +17,12 @@ limitations under the License.
 package dockershim
 
 import (
+	"bytes"
+	"crypto/md5"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
@@ -28,14 +33,21 @@ import (
 	dockernat "github.com/docker/go-connections/nat"
 	"github.com/golang/glog"
 
+	"k8s.io/kubernetes/pkg/api/v1"
 	v1helper "k8s.io/kubernetes/pkg/api/v1/helper"
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
 	"k8s.io/kubernetes/pkg/kubelet/dockertools"
 	"k8s.io/kubernetes/pkg/kubelet/types"
+	"k8s.io/kubernetes/pkg/security/apparmor"
 )
 
 const (
 	annotationPrefix = "annotation."
+
+	// Docker changed the API for specifying options in v1.11
+	securityOptSeparatorChangeVersion = "1.23.0" // Corresponds to docker 1.11.x
+	securityOptSeparatorOld           = ':'
+	securityOptSeparatorNew           = '='
 )
 
 var (
@@ -43,7 +55,9 @@ var (
 
 	// Docker changes the security option separator from ':' to '=' in the 1.23
 	// API version.
-	optsSeparatorChangeVersion = semver.MustParse(dockertools.SecurityOptSeparatorChangeVersion)
+	optsSeparatorChangeVersion = semver.MustParse(securityOptSeparatorChangeVersion)
+
+	defaultSeccompOpt = []dockerOpt{{"seccomp", "unconfined", ""}}
 )
 
 // generateEnvList converts KeyValue list to a list of strings, in the form of
@@ -181,17 +195,57 @@ func makePortsAndBindings(pm []*runtimeapi.PortMapping) (map[dockernat.Port]stru
 	return exposedPorts, portBindings
 }
 
+func getSeccompDockerOpts(annotations map[string]string, ctrName, profileRoot string) ([]dockerOpt, error) {
+	profile, profileOK := annotations[v1.SeccompContainerAnnotationKeyPrefix+ctrName]
+	if !profileOK {
+		// try the pod profile
+		profile, profileOK = annotations[v1.SeccompPodAnnotationKey]
+		if !profileOK {
+			// return early the default
+			return defaultSeccompOpt, nil
+		}
+	}
+
+	if profile == "unconfined" {
+		// return early the default
+		return defaultSeccompOpt, nil
+	}
+
+	if profile == "docker/default" {
+		// return nil so docker will load the default seccomp profile
+		return nil, nil
+	}
+
+	if !strings.HasPrefix(profile, "localhost/") {
+		return nil, fmt.Errorf("unknown seccomp profile option: %s", profile)
+	}
+
+	name := strings.TrimPrefix(profile, "localhost/") // by pod annotation validation, name is a valid subpath
+	fname := filepath.Join(profileRoot, filepath.FromSlash(name))
+	file, err := ioutil.ReadFile(fname)
+	if err != nil {
+		return nil, fmt.Errorf("cannot load seccomp profile %q: %v", name, err)
+	}
+
+	b := bytes.NewBuffer(nil)
+	if err := json.Compact(b, file); err != nil {
+		return nil, err
+	}
+	// Rather than the full profile, just put the filename & md5sum in the event log.
+	msg := fmt.Sprintf("%s(md5:%x)", name, md5.Sum(file))
+
+	return []dockerOpt{{"seccomp", b.String(), msg}}, nil
+}
+
 // getSeccompSecurityOpts gets container seccomp options from container and sandbox
 // config, currently from sandbox annotations.
 // It is an experimental feature and may be promoted to official runtime api in the future.
 func getSeccompSecurityOpts(containerName string, sandboxConfig *runtimeapi.PodSandboxConfig, seccompProfileRoot string, separator rune) ([]string, error) {
-	seccompOpts, err := dockertools.GetSeccompOpts(sandboxConfig.GetAnnotations(), containerName, seccompProfileRoot)
+	seccompOpts, err := getSeccompDockerOpts(sandboxConfig.GetAnnotations(), containerName, seccompProfileRoot)
 	if err != nil {
 		return nil, err
 	}
-
+	return fmtDockerOpts(seccompOpts, separator), nil
-	fmtOpts := dockertools.FmtDockerOpts(seccompOpts, separator)
-	return fmtOpts, nil
 }
 
 // getApparmorSecurityOpts gets apparmor options from container config.
@@ -200,12 +254,12 @@ func getApparmorSecurityOpts(sc *runtimeapi.LinuxContainerSecurityContext, separ
 		return nil, nil
 	}
 
-	appArmorOpts, err := dockertools.GetAppArmorOpts(sc.ApparmorProfile)
+	appArmorOpts, err := getAppArmorOpts(sc.ApparmorProfile)
 	if err != nil {
 		return nil, err
 	}
 
-	fmtOpts := dockertools.FmtDockerOpts(appArmorOpts, separator)
+	fmtOpts := fmtDockerOpts(appArmorOpts, separator)
 	return fmtOpts, nil
 }
 
@@ -258,10 +312,23 @@ func (f *dockerFilter) AddLabel(key, value string) {
 	f.Add("label", fmt.Sprintf("%s=%s", key, value))
 }
 
+// parseUserFromImageUser splits the user out of an user:group string.
+func parseUserFromImageUser(id string) string {
+	if id == "" {
+		return id
+	}
+	// split instances where the id may contain user:group
+	if strings.Contains(id, ":") {
+		return strings.Split(id, ":")[0]
+	}
+	// no group, just return the id
+	return id
+}
+
 // getUserFromImageUser gets uid or user name of the image user.
 // If user is numeric, it will be treated as uid; or else, it is treated as user name.
 func getUserFromImageUser(imageUser string) (*int64, string) {
-	user := dockertools.GetUserFromImageUser(imageUser)
+	user := parseUserFromImageUser(imageUser)
 	// return both nil if user is not specified in the image.
 	if user == "" {
 		return nil, ""
@@ -321,9 +388,9 @@ func getSecurityOptSeparator(v *semver.Version) rune {
 	case -1:
 		// Current version is less than the API change version; use the old
 		// separator.
-		return dockertools.SecurityOptSeparatorOld
+		return securityOptSeparatorOld
 	default:
-		return dockertools.SecurityOptSeparatorNew
+		return securityOptSeparatorNew
 	}
 }
 
@@ -342,3 +409,35 @@ func ensureSandboxImageExists(client dockertools.DockerInterface, image string)
 	}
 	return nil
 }
+
+func getAppArmorOpts(profile string) ([]dockerOpt, error) {
+	if profile == "" || profile == apparmor.ProfileRuntimeDefault {
+		// The docker applies the default profile by default.
+		return nil, nil
+	}
+
+	// Assume validation has already happened.
+	profileName := strings.TrimPrefix(profile, apparmor.ProfileNamePrefix)
+	return []dockerOpt{{"apparmor", profileName, ""}}, nil
+}
+
+// fmtDockerOpts formats the docker security options using the given separator.
+func fmtDockerOpts(opts []dockerOpt, sep rune) []string {
+	fmtOpts := make([]string, len(opts))
+	for i, opt := range opts {
+		fmtOpts[i] = fmt.Sprintf("%s%c%s", opt.key, sep, opt.value)
+	}
+	return fmtOpts
+}
+
+type dockerOpt struct {
+	// The key-value pair passed to docker.
+	key, value string
+	// The alternative value to use in log/event messages.
+	msg string
+}
+
+// Expose key/value from dockertools
+func (d dockerOpt) GetKV() (string, string) {
+	return d.key, d.value
+}

pkg/kubelet/dockershim/security_context.go

@@ -26,7 +26,7 @@ import (
 
 	"k8s.io/kubernetes/pkg/api/v1"
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
-	"k8s.io/kubernetes/pkg/kubelet/dockertools/securitycontext"
+	"k8s.io/kubernetes/pkg/kubelet/dockershim/securitycontext"
 	knetwork "k8s.io/kubernetes/pkg/kubelet/network"
 )
 

pkg/kubelet/dockershim/security_context_test.go

@@ -26,7 +26,7 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
-	"k8s.io/kubernetes/pkg/kubelet/dockertools/securitycontext"
+	"k8s.io/kubernetes/pkg/kubelet/dockershim/securitycontext"
 )
 
 func TestModifyContainerConfig(t *testing.T) {

pkg/kubelet/dockershim/securitycontext/doc.go

@@ -15,4 +15,4 @@ limitations under the License.
 */
 
 // Package securitycontext contains security context api implementations
-package securitycontext // import "k8s.io/kubernetes/pkg/kubelet/dockertools/securitycontext"
+package securitycontext // import "k8s.io/kubernetes/pkg/kubelet/dockershim/securitycontext"

pkg/kubelet/dockertools/BUILD

@@ -14,7 +14,6 @@ go_library(
         "docker.go",
         "docker_manager.go",
         "docker_manager_linux.go",
-        "exec.go",
         "fake_docker_client.go",
         "instrumented_docker.go",
         "kube_docker_client.go",
@@ -22,15 +21,11 @@ go_library(
     tags = ["automanaged"],
     deps = [
         "//pkg/api/v1:go_default_library",
-        "//pkg/client/unversioned/remotecommand:go_default_library",
         "//pkg/credentialprovider:go_default_library",
         "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/images:go_default_library",
         "//pkg/kubelet/leaky:go_default_library",
         "//pkg/kubelet/metrics:go_default_library",
-        "//pkg/security/apparmor:go_default_library",
-        "//pkg/util/exec:go_default_library",
-        "//pkg/util/term:go_default_library",
         "//vendor/github.com/docker/distribution/digest:go_default_library",
         "//vendor/github.com/docker/distribution/reference:go_default_library",
         "//vendor/github.com/docker/docker/pkg/jsonmessage:go_default_library",
@@ -40,7 +35,6 @@ go_library(
         "//vendor/github.com/docker/engine-api/types/container:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/golang.org/x/net/context:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//vendor/k8s.io/client-go/util/clock:go_default_library",
@@ -82,9 +76,6 @@ filegroup(
 
 filegroup(
     name = "all-srcs",
-    srcs = [
+    srcs = [":package-srcs"],
-        ":package-srcs",
-        "//pkg/kubelet/dockertools/securitycontext:all-srcs",
-    ],
     tags = ["automanaged"],
 )

pkg/kubelet/dockertools/docker_manager.go

@@ -16,38 +16,8 @@ limitations under the License.
 
 package dockertools
 
-import (
-	"bytes"
-	"crypto/md5"
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"time"
-
-	dockertypes "github.com/docker/engine-api/types"
-	"github.com/golang/glog"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/kubernetes/pkg/api/v1"
-	"k8s.io/kubernetes/pkg/client/unversioned/remotecommand"
-	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
-	"k8s.io/kubernetes/pkg/security/apparmor"
-)
-
 const (
 	DockerType = "docker"
-	dockerDefaultLoggingDriver = "json-file"
-
-	// Docker changed the API for specifying options in v1.11
-	SecurityOptSeparatorChangeVersion = "1.23.0" // Corresponds to docker 1.11.x
-	SecurityOptSeparatorOld           = ':'
-	SecurityOptSeparatorNew           = '='
 
 	// https://docs.docker.com/engine/reference/api/docker_remote_api/
 	// docker version should be at least 1.10.x
@@ -56,304 +26,4 @@ const (
 	statusRunningPrefix = "Up"
 	statusExitedPrefix  = "Exited"
 	statusCreatedPrefix = "Created"
-
-	ndotsDNSOption = "options ndots:5\n"
 )
-
-var (
-	defaultSeccompOpt = []dockerOpt{{"seccomp", "unconfined", ""}}
-)
-
-// GetImageRef returns the image digest if exists, or else returns the image ID.
-// It is exported for reusing in dockershim.
-func GetImageRef(client DockerInterface, image string) (string, error) {
-	img, err := client.InspectImageByRef(image)
-	if err != nil {
-		return "", err
-	}
-	if img == nil {
-		return "", fmt.Errorf("unable to inspect image %s", image)
-	}
-
-	// Returns the digest if it exist.
-	if len(img.RepoDigests) > 0 {
-		return img.RepoDigests[0], nil
-	}
-
-	return img.ID, nil
-}
-
-// Temporarily export this function to share with dockershim.
-// TODO: clean this up.
-func GetContainerLogs(client DockerInterface, pod *v1.Pod, containerID kubecontainer.ContainerID, logOptions *v1.PodLogOptions, stdout, stderr io.Writer, rawTerm bool) error {
-	var since int64
-	if logOptions.SinceSeconds != nil {
-		t := metav1.Now().Add(-time.Duration(*logOptions.SinceSeconds) * time.Second)
-		since = t.Unix()
-	}
-	if logOptions.SinceTime != nil {
-		since = logOptions.SinceTime.Unix()
-	}
-	opts := dockertypes.ContainerLogsOptions{
-		ShowStdout: true,
-		ShowStderr: true,
-		Since:      strconv.FormatInt(since, 10),
-		Timestamps: logOptions.Timestamps,
-		Follow:     logOptions.Follow,
-	}
-	if logOptions.TailLines != nil {
-		opts.Tail = strconv.FormatInt(*logOptions.TailLines, 10)
-	}
-
-	sopts := StreamOptions{
-		OutputStream: stdout,
-		ErrorStream:  stderr,
-		RawTerminal:  rawTerm,
-	}
-	return client.Logs(containerID.ID, opts, sopts)
-}
-
-// Temporarily export this function to share with dockershim.
-// TODO: clean this up.
-func AttachContainer(client DockerInterface, containerID string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error {
-	// Have to start this before the call to client.AttachToContainer because client.AttachToContainer is a blocking
-	// call :-( Otherwise, resize events don't get processed and the terminal never resizes.
-	kubecontainer.HandleResizing(resize, func(size remotecommand.TerminalSize) {
-		client.ResizeContainerTTY(containerID, int(size.Height), int(size.Width))
-	})
-
-	// TODO(random-liu): Do we really use the *Logs* field here?
-	opts := dockertypes.ContainerAttachOptions{
-		Stream: true,
-		Stdin:  stdin != nil,
-		Stdout: stdout != nil,
-		Stderr: stderr != nil,
-	}
-	sopts := StreamOptions{
-		InputStream:  stdin,
-		OutputStream: stdout,
-		ErrorStream:  stderr,
-		RawTerminal:  tty,
-	}
-	return client.AttachToContainer(containerID, opts, sopts)
-}
-
-// Temporarily export this function to share with dockershim.
-func PortForward(client DockerInterface, podInfraContainerID string, port int32, stream io.ReadWriteCloser) error {
-	container, err := client.InspectContainer(podInfraContainerID)
-	if err != nil {
-		return err
-	}
-
-	if !container.State.Running {
-		return fmt.Errorf("container not running (%s)", container.ID)
-	}
-
-	containerPid := container.State.Pid
-	socatPath, lookupErr := exec.LookPath("socat")
-	if lookupErr != nil {
-		return fmt.Errorf("unable to do port forwarding: socat not found.")
-	}
-
-	args := []string{"-t", fmt.Sprintf("%d", containerPid), "-n", socatPath, "-", fmt.Sprintf("TCP4:localhost:%d", port)}
-
-	nsenterPath, lookupErr := exec.LookPath("nsenter")
-	if lookupErr != nil {
-		return fmt.Errorf("unable to do port forwarding: nsenter not found.")
-	}
-
-	commandString := fmt.Sprintf("%s %s", nsenterPath, strings.Join(args, " "))
-	glog.V(4).Infof("executing port forwarding command: %s", commandString)
-
-	command := exec.Command(nsenterPath, args...)
-	command.Stdout = stream
-
-	stderr := new(bytes.Buffer)
-	command.Stderr = stderr
-
-	// If we use Stdin, command.Run() won't return until the goroutine that's copying
-	// from stream finishes. Unfortunately, if you have a client like telnet connected
-	// via port forwarding, as long as the user's telnet client is connected to the user's
-	// local listener that port forwarding sets up, the telnet session never exits. This
-	// means that even if socat has finished running, command.Run() won't ever return
-	// (because the client still has the connection and stream open).
-	//
-	// The work around is to use StdinPipe(), as Wait() (called by Run()) closes the pipe
-	// when the command (socat) exits.
-	inPipe, err := command.StdinPipe()
-	if err != nil {
-		return fmt.Errorf("unable to do port forwarding: error creating stdin pipe: %v", err)
-	}
-	go func() {
-		io.Copy(inPipe, stream)
-		inPipe.Close()
-	}()
-
-	if err := command.Run(); err != nil {
-		return fmt.Errorf("%v: %s", err, stderr.String())
-	}
-
-	return nil
-}
-
-// Temporarily export this function to share with dockershim.
-// TODO: clean this up.
-func GetAppArmorOpts(profile string) ([]dockerOpt, error) {
-	if profile == "" || profile == apparmor.ProfileRuntimeDefault {
-		// The docker applies the default profile by default.
-		return nil, nil
-	}
-
-	// Assume validation has already happened.
-	profileName := strings.TrimPrefix(profile, apparmor.ProfileNamePrefix)
-	return []dockerOpt{{"apparmor", profileName, ""}}, nil
-}
-
-// Temporarily export this function to share with dockershim.
-// TODO: clean this up.
-func GetSeccompOpts(annotations map[string]string, ctrName, profileRoot string) ([]dockerOpt, error) {
-	profile, profileOK := annotations[v1.SeccompContainerAnnotationKeyPrefix+ctrName]
-	if !profileOK {
-		// try the pod profile
-		profile, profileOK = annotations[v1.SeccompPodAnnotationKey]
-		if !profileOK {
-			// return early the default
-			return defaultSeccompOpt, nil
-		}
-	}
-
-	if profile == "unconfined" {
-		// return early the default
-		return defaultSeccompOpt, nil
-	}
-
-	if profile == "docker/default" {
-		// return nil so docker will load the default seccomp profile
-		return nil, nil
-	}
-
-	if !strings.HasPrefix(profile, "localhost/") {
-		return nil, fmt.Errorf("unknown seccomp profile option: %s", profile)
-	}
-
-	name := strings.TrimPrefix(profile, "localhost/") // by pod annotation validation, name is a valid subpath
-	fname := filepath.Join(profileRoot, filepath.FromSlash(name))
-	file, err := ioutil.ReadFile(fname)
-	if err != nil {
-		return nil, fmt.Errorf("cannot load seccomp profile %q: %v", name, err)
-	}
-
-	b := bytes.NewBuffer(nil)
-	if err := json.Compact(b, file); err != nil {
-		return nil, err
-	}
-	// Rather than the full profile, just put the filename & md5sum in the event log.
-	msg := fmt.Sprintf("%s(md5:%x)", name, md5.Sum(file))
-
-	return []dockerOpt{{"seccomp", b.String(), msg}}, nil
-}
-
-// FmtDockerOpts formats the docker security options using the given separator.
-func FmtDockerOpts(opts []dockerOpt, sep rune) []string {
-	fmtOpts := make([]string, len(opts))
-	for i, opt := range opts {
-		fmtOpts[i] = fmt.Sprintf("%s%c%s", opt.key, sep, opt.value)
-	}
-	return fmtOpts
-}
-
-type dockerOpt struct {
-	// The key-value pair passed to docker.
-	key, value string
-	// The alternative value to use in log/event messages.
-	msg string
-}
-
-// Expose key/value from dockertools
-func (d dockerOpt) GetKV() (string, string) {
-	return d.key, d.value
-}
-
-// GetUserFromImageUser splits the user out of an user:group string.
-func GetUserFromImageUser(id string) string {
-	if id == "" {
-		return id
-	}
-	// split instances where the id may contain user:group
-	if strings.Contains(id, ":") {
-		return strings.Split(id, ":")[0]
-	}
-	// no group, just return the id
-	return id
-}
-
-type dockerExitError struct {
-	Inspect *dockertypes.ContainerExecInspect
-}
-
-func (d *dockerExitError) String() string {
-	return d.Error()
-}
-
-func (d *dockerExitError) Error() string {
-	return fmt.Sprintf("Error executing in Docker Container: %d", d.Inspect.ExitCode)
-}
-
-func (d *dockerExitError) Exited() bool {
-	return !d.Inspect.Running
-}
-
-func (d *dockerExitError) ExitStatus() int {
-	return d.Inspect.ExitCode
-}
-
-// RewriteResolvFile rewrites resolv.conf file generated by docker.
-// Exported for reusing in dockershim.
-func RewriteResolvFile(resolvFilePath string, dns []string, dnsSearch []string, useClusterFirstPolicy bool) error {
-	if len(resolvFilePath) == 0 {
-		glog.Errorf("ResolvConfPath is empty.")
-		return nil
-	}
-
-	if _, err := os.Stat(resolvFilePath); os.IsNotExist(err) {
-		return fmt.Errorf("ResolvConfPath %q does not exist", resolvFilePath)
-	}
-
-	var resolvFileContent []string
-
-	for _, srv := range dns {
-		resolvFileContent = append(resolvFileContent, "nameserver "+srv)
-	}
-
-	if len(dnsSearch) > 0 {
-		resolvFileContent = append(resolvFileContent, "search "+strings.Join(dnsSearch, " "))
-	}
-
-	if len(resolvFileContent) > 0 {
-		if useClusterFirstPolicy {
-			resolvFileContent = append(resolvFileContent, ndotsDNSOption)
-		}
-
-		resolvFileContentStr := strings.Join(resolvFileContent, "\n")
-		resolvFileContentStr += "\n"
-
-		glog.V(4).Infof("Will attempt to re-write config file %s with: \n%s", resolvFilePath, resolvFileContent)
-		if err := rewriteFile(resolvFilePath, resolvFileContentStr); err != nil {
-			glog.Errorf("resolv.conf could not be updated: %v", err)
-			return err
-		}
-	}
-
-	return nil
-}
-
-func rewriteFile(filePath, stringToWrite string) error {
-	f, err := os.OpenFile(filePath, os.O_TRUNC|os.O_WRONLY, 0644)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	_, err = f.WriteString(stringToWrite)
-	return err
-}

pkg/kubelet/kubelet.go

@@ -364,17 +364,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 		KernelMemcgNotification:  kubeCfg.ExperimentalKernelMemcgNotification,
 	}
 
-	var dockerExecHandler dockertools.ExecHandler
-	switch kubeCfg.DockerExecHandlerName {
-	case "native":
-		dockerExecHandler = &dockertools.NativeExecHandler{}
-	case "nsenter":
-		dockerExecHandler = &dockertools.NsenterExecHandler{}
-	default:
-		glog.Warningf("Unknown Docker exec handler %q; defaulting to native", kubeCfg.DockerExecHandlerName)
-		dockerExecHandler = &dockertools.NativeExecHandler{}
-	}
-
 	serviceIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
 	if kubeDeps.KubeClient != nil {
 		serviceLW := cache.NewListWatchFromClient(kubeDeps.KubeClient.Core().RESTClient(), "services", metav1.NamespaceAll, fields.Everything())
@@ -556,7 +545,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 			// Create and start the CRI shim running as a grpc server.
 			streamingConfig := getStreamingConfig(kubeCfg, kubeDeps)
 			ds, err := dockershim.NewDockerService(klet.dockerClient, kubeCfg.SeccompProfileRoot, kubeCfg.PodInfraContainerImage,
-				streamingConfig, &pluginSettings, kubeCfg.RuntimeCgroups, kubeCfg.CgroupDriver, dockerExecHandler, dockershimRootDir,
+				streamingConfig, &pluginSettings, kubeCfg.RuntimeCgroups, kubeCfg.CgroupDriver, kubeCfg.DockerExecHandlerName, dockershimRootDir,
 				!kubeCfg.DockerEnableSharedPID)
 			if err != nil {
 				return nil, err