From 93ecaf6812e2c7315970d87136dc149909a05dee Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Mon, 1 May 2017 15:57:19 -0700 Subject: [PATCH 1/4] Move exec.go from dockertools to dockershim --- cmd/kubelet/app/server.go | 14 +------- pkg/kubelet/dockershim/docker_service.go | 13 +++++++- pkg/kubelet/dockershim/docker_streaming.go | 2 +- .../{dockertools => dockershim}/exec.go | 32 ++++++++++++++++--- pkg/kubelet/dockertools/docker_manager.go | 20 ------------ pkg/kubelet/kubelet.go | 13 +------- 6 files changed, 42 insertions(+), 52 deletions(-) rename pkg/kubelet/{dockertools => dockershim}/exec.go (77%) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 92a41383de7..db289e9d0cf 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -939,18 +939,6 @@ func RunDockershim(c *componentconfig.KubeletConfiguration, dockershimRootDir st dockerClient := dockertools.ConnectToDockerOrDie(c.DockerEndpoint, c.RuntimeRequestTimeout.Duration, c.ImagePullProgressDeadline.Duration) - // Initialize docker exec handler. - var dockerExecHandler dockertools.ExecHandler - switch c.DockerExecHandlerName { - case "native": - dockerExecHandler = &dockertools.NativeExecHandler{} - case "nsenter": - dockerExecHandler = &dockertools.NsenterExecHandler{} - default: - glog.Warningf("Unknown Docker exec handler %q; defaulting to native", c.DockerExecHandlerName) - dockerExecHandler = &dockertools.NativeExecHandler{} - } - // Initialize network plugin settings. binDir := c.CNIBinDir if binDir == "" { @@ -976,7 +964,7 @@ func RunDockershim(c *componentconfig.KubeletConfiguration, dockershimRootDir st } ds, err := dockershim.NewDockerService(dockerClient, c.SeccompProfileRoot, c.PodInfraContainerImage, - streamingConfig, &pluginSettings, c.RuntimeCgroups, c.CgroupDriver, dockerExecHandler, dockershimRootDir, + streamingConfig, &pluginSettings, c.RuntimeCgroups, c.CgroupDriver, c.DockerExecHandlerName, dockershimRootDir, !c.DockerEnableSharedPID) if err != nil { return err diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index 44696c59200..c814c735524 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -147,12 +147,23 @@ var internalLabelKeys []string = []string{containerTypeLabelKey, containerLogPat // NOTE: Anything passed to DockerService should be eventually handled in another way when we switch to running the shim as a different process. func NewDockerService(client dockertools.DockerInterface, seccompProfileRoot string, podSandboxImage string, streamingConfig *streaming.Config, - pluginSettings *NetworkPluginSettings, cgroupsName string, kubeCgroupDriver string, execHandler dockertools.ExecHandler, dockershimRootDir string, disableSharedPID bool) (DockerService, error) { + pluginSettings *NetworkPluginSettings, cgroupsName string, kubeCgroupDriver string, execHandlerName, dockershimRootDir string, disableSharedPID bool) (DockerService, error) { c := dockertools.NewInstrumentedDockerInterface(client) checkpointHandler, err := NewPersistentCheckpointHandler(dockershimRootDir) if err != nil { return nil, err } + var execHandler ExecHandler + switch execHandlerName { + case "native": + execHandler = &NativeExecHandler{} + case "nsenter": + execHandler = &NsenterExecHandler{} + default: + glog.Warningf("Unknown Docker exec handler %q; defaulting to native", execHandlerName) + execHandler = &NativeExecHandler{} + } + ds := &dockerService{ seccompProfileRoot: seccompProfileRoot, client: c, diff --git a/pkg/kubelet/dockershim/docker_streaming.go b/pkg/kubelet/dockershim/docker_streaming.go index 42b299fd1ae..3d05e662112 100644 --- a/pkg/kubelet/dockershim/docker_streaming.go +++ b/pkg/kubelet/dockershim/docker_streaming.go @@ -33,7 +33,7 @@ import ( type streamingRuntime struct { client dockertools.DockerInterface - execHandler dockertools.ExecHandler + execHandler ExecHandler } var _ streaming.Runtime = &streamingRuntime{} diff --git a/pkg/kubelet/dockertools/exec.go b/pkg/kubelet/dockershim/exec.go similarity index 77% rename from pkg/kubelet/dockertools/exec.go rename to pkg/kubelet/dockershim/exec.go index 1f72337e57e..aba59ae25f7 100644 --- a/pkg/kubelet/dockertools/exec.go +++ b/pkg/kubelet/dockershim/exec.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package dockertools +package dockershim import ( "fmt" @@ -25,22 +25,44 @@ import ( dockertypes "github.com/docker/engine-api/types" "github.com/golang/glog" + "k8s.io/kubernetes/pkg/client/unversioned/remotecommand" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/kubelet/dockertools" utilexec "k8s.io/kubernetes/pkg/util/exec" "k8s.io/kubernetes/pkg/util/term" ) // ExecHandler knows how to execute a command in a running Docker container. type ExecHandler interface { - ExecInContainer(client DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error + ExecInContainer(client dockertools.DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error } // NsenterExecHandler executes commands in Docker containers using nsenter. type NsenterExecHandler struct{} +type dockerExitError struct { + Inspect *dockertypes.ContainerExecInspect +} + +func (d *dockerExitError) String() string { + return d.Error() +} + +func (d *dockerExitError) Error() string { + return fmt.Sprintf("Error executing in Docker Container: %d", d.Inspect.ExitCode) +} + +func (d *dockerExitError) Exited() bool { + return !d.Inspect.Running +} + +func (d *dockerExitError) ExitStatus() int { + return d.Inspect.ExitCode +} + // TODO should we support nsenter in a container, running with elevated privs and --pid=host? -func (*NsenterExecHandler) ExecInContainer(client DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error { +func (*NsenterExecHandler) ExecInContainer(client dockertools.DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error { nsenter, err := exec.LookPath("nsenter") if err != nil { return fmt.Errorf("exec unavailable - unable to locate nsenter") @@ -111,7 +133,7 @@ func (*NsenterExecHandler) ExecInContainer(client DockerInterface, container *do // NativeExecHandler executes commands in Docker containers using Docker's exec API. type NativeExecHandler struct{} -func (*NativeExecHandler) ExecInContainer(client DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error { +func (*NativeExecHandler) ExecInContainer(client dockertools.DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error { createOpts := dockertypes.ExecConfig{ Cmd: cmd, AttachStdin: stdin != nil, @@ -131,7 +153,7 @@ func (*NativeExecHandler) ExecInContainer(client DockerInterface, container *doc }) startOpts := dockertypes.ExecStartCheck{Detach: false, Tty: tty} - streamOpts := StreamOptions{ + streamOpts := dockertools.StreamOptions{ InputStream: stdin, OutputStream: stdout, ErrorStream: stderr, diff --git a/pkg/kubelet/dockertools/docker_manager.go b/pkg/kubelet/dockertools/docker_manager.go index e2c3976c519..050541e7140 100644 --- a/pkg/kubelet/dockertools/docker_manager.go +++ b/pkg/kubelet/dockertools/docker_manager.go @@ -287,26 +287,6 @@ func GetUserFromImageUser(id string) string { return id } -type dockerExitError struct { - Inspect *dockertypes.ContainerExecInspect -} - -func (d *dockerExitError) String() string { - return d.Error() -} - -func (d *dockerExitError) Error() string { - return fmt.Sprintf("Error executing in Docker Container: %d", d.Inspect.ExitCode) -} - -func (d *dockerExitError) Exited() bool { - return !d.Inspect.Running -} - -func (d *dockerExitError) ExitStatus() int { - return d.Inspect.ExitCode -} - // RewriteResolvFile rewrites resolv.conf file generated by docker. // Exported for reusing in dockershim. func RewriteResolvFile(resolvFilePath string, dns []string, dnsSearch []string, useClusterFirstPolicy bool) error { diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 6a519fdc707..6525f6d2a73 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -364,17 +364,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub KernelMemcgNotification: kubeCfg.ExperimentalKernelMemcgNotification, } - var dockerExecHandler dockertools.ExecHandler - switch kubeCfg.DockerExecHandlerName { - case "native": - dockerExecHandler = &dockertools.NativeExecHandler{} - case "nsenter": - dockerExecHandler = &dockertools.NsenterExecHandler{} - default: - glog.Warningf("Unknown Docker exec handler %q; defaulting to native", kubeCfg.DockerExecHandlerName) - dockerExecHandler = &dockertools.NativeExecHandler{} - } - serviceIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) if kubeDeps.KubeClient != nil { serviceLW := cache.NewListWatchFromClient(kubeDeps.KubeClient.Core().RESTClient(), "services", metav1.NamespaceAll, fields.Everything()) @@ -556,7 +545,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub // Create and start the CRI shim running as a grpc server. streamingConfig := getStreamingConfig(kubeCfg, kubeDeps) ds, err := dockershim.NewDockerService(klet.dockerClient, kubeCfg.SeccompProfileRoot, kubeCfg.PodInfraContainerImage, - streamingConfig, &pluginSettings, kubeCfg.RuntimeCgroups, kubeCfg.CgroupDriver, dockerExecHandler, dockershimRootDir, + streamingConfig, &pluginSettings, kubeCfg.RuntimeCgroups, kubeCfg.CgroupDriver, kubeCfg.DockerExecHandlerName, dockershimRootDir, !kubeCfg.DockerEnableSharedPID) if err != nil { return nil, err From b209f47562f1e4c1113b06d9bb590e877f9de78e Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Mon, 1 May 2017 17:17:19 -0700 Subject: [PATCH 2/4] Move exported constants/functions from dockertools to dockershim Previously we exported many constants and functions in dockertools to share with the dockershim package. This change moves such constants/functions to dockershim and unexport them. This change involves only mechnical changes and should not have any functional impact. --- pkg/kubelet/dockershim/docker_image.go | 22 +- pkg/kubelet/dockershim/docker_service.go | 29 +- pkg/kubelet/dockershim/docker_streaming.go | 88 +++++- pkg/kubelet/dockershim/helpers.go | 119 +++++++- pkg/kubelet/dockertools/docker_manager.go | 312 +-------------------- 5 files changed, 245 insertions(+), 325 deletions(-) diff --git a/pkg/kubelet/dockershim/docker_image.go b/pkg/kubelet/dockershim/docker_image.go index d568c5164c0..7da7ecd4c6a 100644 --- a/pkg/kubelet/dockershim/docker_image.go +++ b/pkg/kubelet/dockershim/docker_image.go @@ -17,6 +17,8 @@ limitations under the License. package dockershim import ( + "fmt" + dockertypes "github.com/docker/engine-api/types" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" "k8s.io/kubernetes/pkg/kubelet/dockertools" @@ -80,7 +82,7 @@ func (ds *dockerService) PullImage(image *runtimeapi.ImageSpec, auth *runtimeapi return "", err } - return dockertools.GetImageRef(ds.client, image.Image) + return getImageRef(ds.client, image.Image) } // RemoveImage removes the image. @@ -101,3 +103,21 @@ func (ds *dockerService) RemoveImage(image *runtimeapi.ImageSpec) error { _, err = ds.client.RemoveImage(image.Image, dockertypes.ImageRemoveOptions{PruneChildren: true}) return err } + +// getImageRef returns the image digest if exists, or else returns the image ID. +func getImageRef(client dockertools.DockerInterface, image string) (string, error) { + img, err := client.InspectImageByRef(image) + if err != nil { + return "", err + } + if img == nil { + return "", fmt.Errorf("unable to inspect image %s", image) + } + + // Returns the digest if it exist. + if len(img.RepoDigests) > 0 { + return img.RepoDigests[0], nil + } + + return img.ID, nil +} diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index c814c735524..7ea60931f96 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -20,12 +20,14 @@ import ( "fmt" "io" "net/http" + "strconv" "time" "github.com/blang/semver" dockertypes "github.com/docker/engine-api/types" "github.com/golang/glog" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/apis/componentconfig" internalapi "k8s.io/kubernetes/pkg/kubelet/api" @@ -489,7 +491,32 @@ func (d *dockerLegacyService) GetContainerLogs(pod *v1.Pod, containerID kubecont if err != nil { return err } - return dockertools.GetContainerLogs(d.client, pod, containerID, logOptions, stdout, stderr, container.Config.Tty) + + var since int64 + if logOptions.SinceSeconds != nil { + t := metav1.Now().Add(-time.Duration(*logOptions.SinceSeconds) * time.Second) + since = t.Unix() + } + if logOptions.SinceTime != nil { + since = logOptions.SinceTime.Unix() + } + opts := dockertypes.ContainerLogsOptions{ + ShowStdout: true, + ShowStderr: true, + Since: strconv.FormatInt(since, 10), + Timestamps: logOptions.Timestamps, + Follow: logOptions.Follow, + } + if logOptions.TailLines != nil { + opts.Tail = strconv.FormatInt(*logOptions.TailLines, 10) + } + + sopts := dockertools.StreamOptions{ + OutputStream: stdout, + ErrorStream: stderr, + RawTerminal: container.Config.Tty, + } + return d.client.Logs(containerID.ID, opts, sopts) } // criSupportedLogDrivers are log drivers supported by native CRI integration. diff --git a/pkg/kubelet/dockershim/docker_streaming.go b/pkg/kubelet/dockershim/docker_streaming.go index 3d05e662112..262cadfee0a 100644 --- a/pkg/kubelet/dockershim/docker_streaming.go +++ b/pkg/kubelet/dockershim/docker_streaming.go @@ -21,11 +21,15 @@ import ( "fmt" "io" "math" + "os/exec" + "strings" "time" dockertypes "github.com/docker/engine-api/types" + "github.com/golang/glog" "k8s.io/kubernetes/pkg/client/unversioned/remotecommand" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockertools" "k8s.io/kubernetes/pkg/kubelet/server/streaming" "k8s.io/kubernetes/pkg/kubelet/util/ioutils" @@ -57,14 +61,14 @@ func (r *streamingRuntime) Attach(containerID string, in io.Reader, out, errw io return err } - return dockertools.AttachContainer(r.client, containerID, in, out, errw, tty, resize) + return attachContainer(r.client, containerID, in, out, errw, tty, resize) } func (r *streamingRuntime) PortForward(podSandboxID string, port int32, stream io.ReadWriteCloser) error { if port < 0 || port > math.MaxUint16 { return fmt.Errorf("invalid port %d", port) } - return dockertools.PortForward(r.client, podSandboxID, port, stream) + return portForward(r.client, podSandboxID, port, stream) } // ExecSync executes a command in the container, and returns the stdout output. @@ -128,3 +132,83 @@ func checkContainerStatus(client dockertools.DockerInterface, containerID string } return container, nil } + +func attachContainer(client dockertools.DockerInterface, containerID string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error { + // Have to start this before the call to client.AttachToContainer because client.AttachToContainer is a blocking + // call :-( Otherwise, resize events don't get processed and the terminal never resizes. + kubecontainer.HandleResizing(resize, func(size remotecommand.TerminalSize) { + client.ResizeContainerTTY(containerID, int(size.Height), int(size.Width)) + }) + + // TODO(random-liu): Do we really use the *Logs* field here? + opts := dockertypes.ContainerAttachOptions{ + Stream: true, + Stdin: stdin != nil, + Stdout: stdout != nil, + Stderr: stderr != nil, + } + sopts := dockertools.StreamOptions{ + InputStream: stdin, + OutputStream: stdout, + ErrorStream: stderr, + RawTerminal: tty, + } + return client.AttachToContainer(containerID, opts, sopts) +} + +func portForward(client dockertools.DockerInterface, podInfraContainerID string, port int32, stream io.ReadWriteCloser) error { + container, err := client.InspectContainer(podInfraContainerID) + if err != nil { + return err + } + + if !container.State.Running { + return fmt.Errorf("container not running (%s)", container.ID) + } + + containerPid := container.State.Pid + socatPath, lookupErr := exec.LookPath("socat") + if lookupErr != nil { + return fmt.Errorf("unable to do port forwarding: socat not found.") + } + + args := []string{"-t", fmt.Sprintf("%d", containerPid), "-n", socatPath, "-", fmt.Sprintf("TCP4:localhost:%d", port)} + + nsenterPath, lookupErr := exec.LookPath("nsenter") + if lookupErr != nil { + return fmt.Errorf("unable to do port forwarding: nsenter not found.") + } + + commandString := fmt.Sprintf("%s %s", nsenterPath, strings.Join(args, " ")) + glog.V(4).Infof("executing port forwarding command: %s", commandString) + + command := exec.Command(nsenterPath, args...) + command.Stdout = stream + + stderr := new(bytes.Buffer) + command.Stderr = stderr + + // If we use Stdin, command.Run() won't return until the goroutine that's copying + // from stream finishes. Unfortunately, if you have a client like telnet connected + // via port forwarding, as long as the user's telnet client is connected to the user's + // local listener that port forwarding sets up, the telnet session never exits. This + // means that even if socat has finished running, command.Run() won't ever return + // (because the client still has the connection and stream open). + // + // The work around is to use StdinPipe(), as Wait() (called by Run()) closes the pipe + // when the command (socat) exits. + inPipe, err := command.StdinPipe() + if err != nil { + return fmt.Errorf("unable to do port forwarding: error creating stdin pipe: %v", err) + } + go func() { + io.Copy(inPipe, stream) + inPipe.Close() + }() + + if err := command.Run(); err != nil { + return fmt.Errorf("%v: %s", err, stderr.String()) + } + + return nil +} diff --git a/pkg/kubelet/dockershim/helpers.go b/pkg/kubelet/dockershim/helpers.go index def9df08c2c..0927ad35b45 100644 --- a/pkg/kubelet/dockershim/helpers.go +++ b/pkg/kubelet/dockershim/helpers.go @@ -17,7 +17,12 @@ limitations under the License. package dockershim import ( + "bytes" + "crypto/md5" + "encoding/json" "fmt" + "io/ioutil" + "path/filepath" "regexp" "strconv" "strings" @@ -28,14 +33,21 @@ import ( dockernat "github.com/docker/go-connections/nat" "github.com/golang/glog" + "k8s.io/kubernetes/pkg/api/v1" v1helper "k8s.io/kubernetes/pkg/api/v1/helper" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" "k8s.io/kubernetes/pkg/kubelet/dockertools" "k8s.io/kubernetes/pkg/kubelet/types" + "k8s.io/kubernetes/pkg/security/apparmor" ) const ( annotationPrefix = "annotation." + + // Docker changed the API for specifying options in v1.11 + securityOptSeparatorChangeVersion = "1.23.0" // Corresponds to docker 1.11.x + securityOptSeparatorOld = ':' + securityOptSeparatorNew = '=' ) var ( @@ -43,7 +55,9 @@ var ( // Docker changes the security option separator from ':' to '=' in the 1.23 // API version. - optsSeparatorChangeVersion = semver.MustParse(dockertools.SecurityOptSeparatorChangeVersion) + optsSeparatorChangeVersion = semver.MustParse(securityOptSeparatorChangeVersion) + + defaultSeccompOpt = []dockerOpt{{"seccomp", "unconfined", ""}} ) // generateEnvList converts KeyValue list to a list of strings, in the form of @@ -181,17 +195,57 @@ func makePortsAndBindings(pm []*runtimeapi.PortMapping) (map[dockernat.Port]stru return exposedPorts, portBindings } +func getSeccompDockerOpts(annotations map[string]string, ctrName, profileRoot string) ([]dockerOpt, error) { + profile, profileOK := annotations[v1.SeccompContainerAnnotationKeyPrefix+ctrName] + if !profileOK { + // try the pod profile + profile, profileOK = annotations[v1.SeccompPodAnnotationKey] + if !profileOK { + // return early the default + return defaultSeccompOpt, nil + } + } + + if profile == "unconfined" { + // return early the default + return defaultSeccompOpt, nil + } + + if profile == "docker/default" { + // return nil so docker will load the default seccomp profile + return nil, nil + } + + if !strings.HasPrefix(profile, "localhost/") { + return nil, fmt.Errorf("unknown seccomp profile option: %s", profile) + } + + name := strings.TrimPrefix(profile, "localhost/") // by pod annotation validation, name is a valid subpath + fname := filepath.Join(profileRoot, filepath.FromSlash(name)) + file, err := ioutil.ReadFile(fname) + if err != nil { + return nil, fmt.Errorf("cannot load seccomp profile %q: %v", name, err) + } + + b := bytes.NewBuffer(nil) + if err := json.Compact(b, file); err != nil { + return nil, err + } + // Rather than the full profile, just put the filename & md5sum in the event log. + msg := fmt.Sprintf("%s(md5:%x)", name, md5.Sum(file)) + + return []dockerOpt{{"seccomp", b.String(), msg}}, nil +} + // getSeccompSecurityOpts gets container seccomp options from container and sandbox // config, currently from sandbox annotations. // It is an experimental feature and may be promoted to official runtime api in the future. func getSeccompSecurityOpts(containerName string, sandboxConfig *runtimeapi.PodSandboxConfig, seccompProfileRoot string, separator rune) ([]string, error) { - seccompOpts, err := dockertools.GetSeccompOpts(sandboxConfig.GetAnnotations(), containerName, seccompProfileRoot) + seccompOpts, err := getSeccompDockerOpts(sandboxConfig.GetAnnotations(), containerName, seccompProfileRoot) if err != nil { return nil, err } - - fmtOpts := dockertools.FmtDockerOpts(seccompOpts, separator) - return fmtOpts, nil + return fmtDockerOpts(seccompOpts, separator), nil } // getApparmorSecurityOpts gets apparmor options from container config. @@ -200,12 +254,12 @@ func getApparmorSecurityOpts(sc *runtimeapi.LinuxContainerSecurityContext, separ return nil, nil } - appArmorOpts, err := dockertools.GetAppArmorOpts(sc.ApparmorProfile) + appArmorOpts, err := getAppArmorOpts(sc.ApparmorProfile) if err != nil { return nil, err } - fmtOpts := dockertools.FmtDockerOpts(appArmorOpts, separator) + fmtOpts := fmtDockerOpts(appArmorOpts, separator) return fmtOpts, nil } @@ -258,10 +312,23 @@ func (f *dockerFilter) AddLabel(key, value string) { f.Add("label", fmt.Sprintf("%s=%s", key, value)) } +// parseUserFromImageUser splits the user out of an user:group string. +func parseUserFromImageUser(id string) string { + if id == "" { + return id + } + // split instances where the id may contain user:group + if strings.Contains(id, ":") { + return strings.Split(id, ":")[0] + } + // no group, just return the id + return id +} + // getUserFromImageUser gets uid or user name of the image user. // If user is numeric, it will be treated as uid; or else, it is treated as user name. func getUserFromImageUser(imageUser string) (*int64, string) { - user := dockertools.GetUserFromImageUser(imageUser) + user := parseUserFromImageUser(imageUser) // return both nil if user is not specified in the image. if user == "" { return nil, "" @@ -321,9 +388,9 @@ func getSecurityOptSeparator(v *semver.Version) rune { case -1: // Current version is less than the API change version; use the old // separator. - return dockertools.SecurityOptSeparatorOld + return securityOptSeparatorOld default: - return dockertools.SecurityOptSeparatorNew + return securityOptSeparatorNew } } @@ -342,3 +409,35 @@ func ensureSandboxImageExists(client dockertools.DockerInterface, image string) } return nil } + +func getAppArmorOpts(profile string) ([]dockerOpt, error) { + if profile == "" || profile == apparmor.ProfileRuntimeDefault { + // The docker applies the default profile by default. + return nil, nil + } + + // Assume validation has already happened. + profileName := strings.TrimPrefix(profile, apparmor.ProfileNamePrefix) + return []dockerOpt{{"apparmor", profileName, ""}}, nil +} + +// fmtDockerOpts formats the docker security options using the given separator. +func fmtDockerOpts(opts []dockerOpt, sep rune) []string { + fmtOpts := make([]string, len(opts)) + for i, opt := range opts { + fmtOpts[i] = fmt.Sprintf("%s%c%s", opt.key, sep, opt.value) + } + return fmtOpts +} + +type dockerOpt struct { + // The key-value pair passed to docker. + key, value string + // The alternative value to use in log/event messages. + msg string +} + +// Expose key/value from dockertools +func (d dockerOpt) GetKV() (string, string) { + return d.key, d.value +} diff --git a/pkg/kubelet/dockertools/docker_manager.go b/pkg/kubelet/dockertools/docker_manager.go index 050541e7140..aaf4858456e 100644 --- a/pkg/kubelet/dockertools/docker_manager.go +++ b/pkg/kubelet/dockertools/docker_manager.go @@ -16,38 +16,8 @@ limitations under the License. package dockertools -import ( - "bytes" - "crypto/md5" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - "time" - - dockertypes "github.com/docker/engine-api/types" - "github.com/golang/glog" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kubernetes/pkg/api/v1" - "k8s.io/kubernetes/pkg/client/unversioned/remotecommand" - kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" - "k8s.io/kubernetes/pkg/security/apparmor" -) - const ( - DockerType = "docker" - dockerDefaultLoggingDriver = "json-file" - - // Docker changed the API for specifying options in v1.11 - SecurityOptSeparatorChangeVersion = "1.23.0" // Corresponds to docker 1.11.x - SecurityOptSeparatorOld = ':' - SecurityOptSeparatorNew = '=' + DockerType = "docker" // https://docs.docker.com/engine/reference/api/docker_remote_api/ // docker version should be at least 1.10.x @@ -56,284 +26,4 @@ const ( statusRunningPrefix = "Up" statusExitedPrefix = "Exited" statusCreatedPrefix = "Created" - - ndotsDNSOption = "options ndots:5\n" ) - -var ( - defaultSeccompOpt = []dockerOpt{{"seccomp", "unconfined", ""}} -) - -// GetImageRef returns the image digest if exists, or else returns the image ID. -// It is exported for reusing in dockershim. -func GetImageRef(client DockerInterface, image string) (string, error) { - img, err := client.InspectImageByRef(image) - if err != nil { - return "", err - } - if img == nil { - return "", fmt.Errorf("unable to inspect image %s", image) - } - - // Returns the digest if it exist. - if len(img.RepoDigests) > 0 { - return img.RepoDigests[0], nil - } - - return img.ID, nil -} - -// Temporarily export this function to share with dockershim. -// TODO: clean this up. -func GetContainerLogs(client DockerInterface, pod *v1.Pod, containerID kubecontainer.ContainerID, logOptions *v1.PodLogOptions, stdout, stderr io.Writer, rawTerm bool) error { - var since int64 - if logOptions.SinceSeconds != nil { - t := metav1.Now().Add(-time.Duration(*logOptions.SinceSeconds) * time.Second) - since = t.Unix() - } - if logOptions.SinceTime != nil { - since = logOptions.SinceTime.Unix() - } - opts := dockertypes.ContainerLogsOptions{ - ShowStdout: true, - ShowStderr: true, - Since: strconv.FormatInt(since, 10), - Timestamps: logOptions.Timestamps, - Follow: logOptions.Follow, - } - if logOptions.TailLines != nil { - opts.Tail = strconv.FormatInt(*logOptions.TailLines, 10) - } - - sopts := StreamOptions{ - OutputStream: stdout, - ErrorStream: stderr, - RawTerminal: rawTerm, - } - return client.Logs(containerID.ID, opts, sopts) -} - -// Temporarily export this function to share with dockershim. -// TODO: clean this up. -func AttachContainer(client DockerInterface, containerID string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error { - // Have to start this before the call to client.AttachToContainer because client.AttachToContainer is a blocking - // call :-( Otherwise, resize events don't get processed and the terminal never resizes. - kubecontainer.HandleResizing(resize, func(size remotecommand.TerminalSize) { - client.ResizeContainerTTY(containerID, int(size.Height), int(size.Width)) - }) - - // TODO(random-liu): Do we really use the *Logs* field here? - opts := dockertypes.ContainerAttachOptions{ - Stream: true, - Stdin: stdin != nil, - Stdout: stdout != nil, - Stderr: stderr != nil, - } - sopts := StreamOptions{ - InputStream: stdin, - OutputStream: stdout, - ErrorStream: stderr, - RawTerminal: tty, - } - return client.AttachToContainer(containerID, opts, sopts) -} - -// Temporarily export this function to share with dockershim. -func PortForward(client DockerInterface, podInfraContainerID string, port int32, stream io.ReadWriteCloser) error { - container, err := client.InspectContainer(podInfraContainerID) - if err != nil { - return err - } - - if !container.State.Running { - return fmt.Errorf("container not running (%s)", container.ID) - } - - containerPid := container.State.Pid - socatPath, lookupErr := exec.LookPath("socat") - if lookupErr != nil { - return fmt.Errorf("unable to do port forwarding: socat not found.") - } - - args := []string{"-t", fmt.Sprintf("%d", containerPid), "-n", socatPath, "-", fmt.Sprintf("TCP4:localhost:%d", port)} - - nsenterPath, lookupErr := exec.LookPath("nsenter") - if lookupErr != nil { - return fmt.Errorf("unable to do port forwarding: nsenter not found.") - } - - commandString := fmt.Sprintf("%s %s", nsenterPath, strings.Join(args, " ")) - glog.V(4).Infof("executing port forwarding command: %s", commandString) - - command := exec.Command(nsenterPath, args...) - command.Stdout = stream - - stderr := new(bytes.Buffer) - command.Stderr = stderr - - // If we use Stdin, command.Run() won't return until the goroutine that's copying - // from stream finishes. Unfortunately, if you have a client like telnet connected - // via port forwarding, as long as the user's telnet client is connected to the user's - // local listener that port forwarding sets up, the telnet session never exits. This - // means that even if socat has finished running, command.Run() won't ever return - // (because the client still has the connection and stream open). - // - // The work around is to use StdinPipe(), as Wait() (called by Run()) closes the pipe - // when the command (socat) exits. - inPipe, err := command.StdinPipe() - if err != nil { - return fmt.Errorf("unable to do port forwarding: error creating stdin pipe: %v", err) - } - go func() { - io.Copy(inPipe, stream) - inPipe.Close() - }() - - if err := command.Run(); err != nil { - return fmt.Errorf("%v: %s", err, stderr.String()) - } - - return nil -} - -// Temporarily export this function to share with dockershim. -// TODO: clean this up. -func GetAppArmorOpts(profile string) ([]dockerOpt, error) { - if profile == "" || profile == apparmor.ProfileRuntimeDefault { - // The docker applies the default profile by default. - return nil, nil - } - - // Assume validation has already happened. - profileName := strings.TrimPrefix(profile, apparmor.ProfileNamePrefix) - return []dockerOpt{{"apparmor", profileName, ""}}, nil -} - -// Temporarily export this function to share with dockershim. -// TODO: clean this up. -func GetSeccompOpts(annotations map[string]string, ctrName, profileRoot string) ([]dockerOpt, error) { - profile, profileOK := annotations[v1.SeccompContainerAnnotationKeyPrefix+ctrName] - if !profileOK { - // try the pod profile - profile, profileOK = annotations[v1.SeccompPodAnnotationKey] - if !profileOK { - // return early the default - return defaultSeccompOpt, nil - } - } - - if profile == "unconfined" { - // return early the default - return defaultSeccompOpt, nil - } - - if profile == "docker/default" { - // return nil so docker will load the default seccomp profile - return nil, nil - } - - if !strings.HasPrefix(profile, "localhost/") { - return nil, fmt.Errorf("unknown seccomp profile option: %s", profile) - } - - name := strings.TrimPrefix(profile, "localhost/") // by pod annotation validation, name is a valid subpath - fname := filepath.Join(profileRoot, filepath.FromSlash(name)) - file, err := ioutil.ReadFile(fname) - if err != nil { - return nil, fmt.Errorf("cannot load seccomp profile %q: %v", name, err) - } - - b := bytes.NewBuffer(nil) - if err := json.Compact(b, file); err != nil { - return nil, err - } - // Rather than the full profile, just put the filename & md5sum in the event log. - msg := fmt.Sprintf("%s(md5:%x)", name, md5.Sum(file)) - - return []dockerOpt{{"seccomp", b.String(), msg}}, nil -} - -// FmtDockerOpts formats the docker security options using the given separator. -func FmtDockerOpts(opts []dockerOpt, sep rune) []string { - fmtOpts := make([]string, len(opts)) - for i, opt := range opts { - fmtOpts[i] = fmt.Sprintf("%s%c%s", opt.key, sep, opt.value) - } - return fmtOpts -} - -type dockerOpt struct { - // The key-value pair passed to docker. - key, value string - // The alternative value to use in log/event messages. - msg string -} - -// Expose key/value from dockertools -func (d dockerOpt) GetKV() (string, string) { - return d.key, d.value -} - -// GetUserFromImageUser splits the user out of an user:group string. -func GetUserFromImageUser(id string) string { - if id == "" { - return id - } - // split instances where the id may contain user:group - if strings.Contains(id, ":") { - return strings.Split(id, ":")[0] - } - // no group, just return the id - return id -} - -// RewriteResolvFile rewrites resolv.conf file generated by docker. -// Exported for reusing in dockershim. -func RewriteResolvFile(resolvFilePath string, dns []string, dnsSearch []string, useClusterFirstPolicy bool) error { - if len(resolvFilePath) == 0 { - glog.Errorf("ResolvConfPath is empty.") - return nil - } - - if _, err := os.Stat(resolvFilePath); os.IsNotExist(err) { - return fmt.Errorf("ResolvConfPath %q does not exist", resolvFilePath) - } - - var resolvFileContent []string - - for _, srv := range dns { - resolvFileContent = append(resolvFileContent, "nameserver "+srv) - } - - if len(dnsSearch) > 0 { - resolvFileContent = append(resolvFileContent, "search "+strings.Join(dnsSearch, " ")) - } - - if len(resolvFileContent) > 0 { - if useClusterFirstPolicy { - resolvFileContent = append(resolvFileContent, ndotsDNSOption) - } - - resolvFileContentStr := strings.Join(resolvFileContent, "\n") - resolvFileContentStr += "\n" - - glog.V(4).Infof("Will attempt to re-write config file %s with: \n%s", resolvFilePath, resolvFileContent) - if err := rewriteFile(resolvFilePath, resolvFileContentStr); err != nil { - glog.Errorf("resolv.conf could not be updated: %v", err) - return err - } - } - - return nil -} - -func rewriteFile(filePath, stringToWrite string) error { - f, err := os.OpenFile(filePath, os.O_TRUNC|os.O_WRONLY, 0644) - if err != nil { - return err - } - defer f.Close() - - _, err = f.WriteString(stringToWrite) - return err -} From 951b2d922b24d0c002c92f7b62d9863f0acb02fe Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Mon, 1 May 2017 17:31:11 -0700 Subject: [PATCH 3/4] move securitycontext from dockertools to dockershim --- pkg/kubelet/dockershim/security_context.go | 2 +- pkg/kubelet/dockershim/security_context_test.go | 2 +- pkg/kubelet/{dockertools => dockershim}/securitycontext/BUILD | 0 pkg/kubelet/{dockertools => dockershim}/securitycontext/doc.go | 2 +- pkg/kubelet/{dockertools => dockershim}/securitycontext/fake.go | 0 .../{dockertools => dockershim}/securitycontext/provider.go | 0 .../securitycontext/provider_test.go | 0 .../{dockertools => dockershim}/securitycontext/types.go | 0 pkg/kubelet/{dockertools => dockershim}/securitycontext/util.go | 0 9 files changed, 3 insertions(+), 3 deletions(-) rename pkg/kubelet/{dockertools => dockershim}/securitycontext/BUILD (100%) rename pkg/kubelet/{dockertools => dockershim}/securitycontext/doc.go (95%) rename pkg/kubelet/{dockertools => dockershim}/securitycontext/fake.go (100%) rename pkg/kubelet/{dockertools => dockershim}/securitycontext/provider.go (100%) rename pkg/kubelet/{dockertools => dockershim}/securitycontext/provider_test.go (100%) rename pkg/kubelet/{dockertools => dockershim}/securitycontext/types.go (100%) rename pkg/kubelet/{dockertools => dockershim}/securitycontext/util.go (100%) diff --git a/pkg/kubelet/dockershim/security_context.go b/pkg/kubelet/dockershim/security_context.go index f683110270c..f116675126f 100644 --- a/pkg/kubelet/dockershim/security_context.go +++ b/pkg/kubelet/dockershim/security_context.go @@ -26,7 +26,7 @@ import ( "k8s.io/kubernetes/pkg/api/v1" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" - "k8s.io/kubernetes/pkg/kubelet/dockertools/securitycontext" + "k8s.io/kubernetes/pkg/kubelet/dockershim/securitycontext" knetwork "k8s.io/kubernetes/pkg/kubelet/network" ) diff --git a/pkg/kubelet/dockershim/security_context_test.go b/pkg/kubelet/dockershim/security_context_test.go index 1074ee411a2..001dba61869 100644 --- a/pkg/kubelet/dockershim/security_context_test.go +++ b/pkg/kubelet/dockershim/security_context_test.go @@ -26,7 +26,7 @@ import ( "github.com/stretchr/testify/assert" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" - "k8s.io/kubernetes/pkg/kubelet/dockertools/securitycontext" + "k8s.io/kubernetes/pkg/kubelet/dockershim/securitycontext" ) func TestModifyContainerConfig(t *testing.T) { diff --git a/pkg/kubelet/dockertools/securitycontext/BUILD b/pkg/kubelet/dockershim/securitycontext/BUILD similarity index 100% rename from pkg/kubelet/dockertools/securitycontext/BUILD rename to pkg/kubelet/dockershim/securitycontext/BUILD diff --git a/pkg/kubelet/dockertools/securitycontext/doc.go b/pkg/kubelet/dockershim/securitycontext/doc.go similarity index 95% rename from pkg/kubelet/dockertools/securitycontext/doc.go rename to pkg/kubelet/dockershim/securitycontext/doc.go index 72fde215674..dd9a0a2291d 100644 --- a/pkg/kubelet/dockertools/securitycontext/doc.go +++ b/pkg/kubelet/dockershim/securitycontext/doc.go @@ -15,4 +15,4 @@ limitations under the License. */ // Package securitycontext contains security context api implementations -package securitycontext // import "k8s.io/kubernetes/pkg/kubelet/dockertools/securitycontext" +package securitycontext // import "k8s.io/kubernetes/pkg/kubelet/dockershim/securitycontext" diff --git a/pkg/kubelet/dockertools/securitycontext/fake.go b/pkg/kubelet/dockershim/securitycontext/fake.go similarity index 100% rename from pkg/kubelet/dockertools/securitycontext/fake.go rename to pkg/kubelet/dockershim/securitycontext/fake.go diff --git a/pkg/kubelet/dockertools/securitycontext/provider.go b/pkg/kubelet/dockershim/securitycontext/provider.go similarity index 100% rename from pkg/kubelet/dockertools/securitycontext/provider.go rename to pkg/kubelet/dockershim/securitycontext/provider.go diff --git a/pkg/kubelet/dockertools/securitycontext/provider_test.go b/pkg/kubelet/dockershim/securitycontext/provider_test.go similarity index 100% rename from pkg/kubelet/dockertools/securitycontext/provider_test.go rename to pkg/kubelet/dockershim/securitycontext/provider_test.go diff --git a/pkg/kubelet/dockertools/securitycontext/types.go b/pkg/kubelet/dockershim/securitycontext/types.go similarity index 100% rename from pkg/kubelet/dockertools/securitycontext/types.go rename to pkg/kubelet/dockershim/securitycontext/types.go diff --git a/pkg/kubelet/dockertools/securitycontext/util.go b/pkg/kubelet/dockershim/securitycontext/util.go similarity index 100% rename from pkg/kubelet/dockertools/securitycontext/util.go rename to pkg/kubelet/dockershim/securitycontext/util.go From c35c00f3f152ccc9da8057499faf27c673a03ca2 Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Mon, 1 May 2017 17:32:58 -0700 Subject: [PATCH 4/4] update bazel --- pkg/kubelet/dockershim/BUILD | 10 ++++++++-- pkg/kubelet/dockertools/BUILD | 11 +---------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/pkg/kubelet/dockershim/BUILD b/pkg/kubelet/dockershim/BUILD index 4b23c221e9e..10d0045f0a4 100644 --- a/pkg/kubelet/dockershim/BUILD +++ b/pkg/kubelet/dockershim/BUILD @@ -21,6 +21,7 @@ go_library( "docker_sandbox.go", "docker_service.go", "docker_streaming.go", + "exec.go", "helpers.go", "naming.go", "security_context.go", @@ -37,8 +38,8 @@ go_library( "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/dockershim/cm:go_default_library", "//pkg/kubelet/dockershim/errors:go_default_library", + "//pkg/kubelet/dockershim/securitycontext:go_default_library", "//pkg/kubelet/dockertools:go_default_library", - "//pkg/kubelet/dockertools/securitycontext:go_default_library", "//pkg/kubelet/leaky:go_default_library", "//pkg/kubelet/network:go_default_library", "//pkg/kubelet/network/cni:go_default_library", @@ -49,7 +50,10 @@ go_library( "//pkg/kubelet/types:go_default_library", "//pkg/kubelet/util/cache:go_default_library", "//pkg/kubelet/util/ioutils:go_default_library", + "//pkg/security/apparmor:go_default_library", + "//pkg/util/exec:go_default_library", "//pkg/util/hash:go_default_library", + "//pkg/util/term:go_default_library", "//vendor/github.com/blang/semver:go_default_library", "//vendor/github.com/docker/engine-api/types:go_default_library", "//vendor/github.com/docker/engine-api/types/container:go_default_library", @@ -57,6 +61,7 @@ go_library( "//vendor/github.com/docker/engine-api/types/strslice:go_default_library", "//vendor/github.com/docker/go-connections/nat:go_default_library", "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", @@ -86,9 +91,9 @@ go_test( "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/container/testing:go_default_library", "//pkg/kubelet/dockershim/errors:go_default_library", + "//pkg/kubelet/dockershim/securitycontext:go_default_library", "//pkg/kubelet/dockershim/testing:go_default_library", "//pkg/kubelet/dockertools:go_default_library", - "//pkg/kubelet/dockertools/securitycontext:go_default_library", "//pkg/kubelet/network:go_default_library", "//pkg/kubelet/network/testing:go_default_library", "//pkg/kubelet/types:go_default_library", @@ -119,6 +124,7 @@ filegroup( "//pkg/kubelet/dockershim/cm:all-srcs", "//pkg/kubelet/dockershim/errors:all-srcs", "//pkg/kubelet/dockershim/remote:all-srcs", + "//pkg/kubelet/dockershim/securitycontext:all-srcs", "//pkg/kubelet/dockershim/testing:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/kubelet/dockertools/BUILD b/pkg/kubelet/dockertools/BUILD index 20d0bcbb43c..400f22b0aaa 100644 --- a/pkg/kubelet/dockertools/BUILD +++ b/pkg/kubelet/dockertools/BUILD @@ -14,7 +14,6 @@ go_library( "docker.go", "docker_manager.go", "docker_manager_linux.go", - "exec.go", "fake_docker_client.go", "instrumented_docker.go", "kube_docker_client.go", @@ -22,15 +21,11 @@ go_library( tags = ["automanaged"], deps = [ "//pkg/api/v1:go_default_library", - "//pkg/client/unversioned/remotecommand:go_default_library", "//pkg/credentialprovider:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/images:go_default_library", "//pkg/kubelet/leaky:go_default_library", "//pkg/kubelet/metrics:go_default_library", - "//pkg/security/apparmor:go_default_library", - "//pkg/util/exec:go_default_library", - "//pkg/util/term:go_default_library", "//vendor/github.com/docker/distribution/digest:go_default_library", "//vendor/github.com/docker/distribution/reference:go_default_library", "//vendor/github.com/docker/docker/pkg/jsonmessage:go_default_library", @@ -40,7 +35,6 @@ go_library( "//vendor/github.com/docker/engine-api/types/container:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/golang.org/x/net/context:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/k8s.io/client-go/util/clock:go_default_library", @@ -82,9 +76,6 @@ filegroup( filegroup( name = "all-srcs", - srcs = [ - ":package-srcs", - "//pkg/kubelet/dockertools/securitycontext:all-srcs", - ], + srcs = [":package-srcs"], tags = ["automanaged"], )