diff --git a/contrib/mesos/pkg/executor/executor.go b/contrib/mesos/pkg/executor/executor.go index c32083c617c..3dfe68d14d2 100644 --- a/contrib/mesos/pkg/executor/executor.go +++ b/contrib/mesos/pkg/executor/executor.go @@ -28,7 +28,7 @@ import ( clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - "github.com/fsouza/go-dockerclient" + dockertypes "github.com/docker/engine-api/types" "github.com/gogo/protobuf/proto" log "github.com/golang/glog" bindings "github.com/mesos/mesos-go/executor" @@ -655,14 +655,13 @@ func (k *Executor) doShutdown(driver bindings.ExecutorDriver) { // Destroy existing k8s containers func (k *Executor) killKubeletContainers() { if containers, err := dockertools.GetKubeletDockerContainers(k.dockerClient, true); err == nil { - opts := docker.RemoveContainerOptions{ + opts := dockertypes.ContainerRemoveOptions{ RemoveVolumes: true, Force: true, } for _, container := range containers { - opts.ID = container.ID - log.V(2).Infof("Removing container: %v", opts.ID) - if err := k.dockerClient.RemoveContainer(opts); err != nil { + log.V(2).Infof("Removing container: %v", container.ID) + if err := k.dockerClient.RemoveContainer(container.ID, opts); err != nil { log.Warning(err) } } diff --git a/pkg/kubelet/dockertools/container_gc.go b/pkg/kubelet/dockertools/container_gc.go index fae2194b067..22319f7fdbe 100644 --- a/pkg/kubelet/dockertools/container_gc.go +++ b/pkg/kubelet/dockertools/container_gc.go @@ -24,7 +24,7 @@ import ( "sort" "time" - docker "github.com/fsouza/go-dockerclient" + dockertypes "github.com/docker/engine-api/types" "github.com/golang/glog" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/types" @@ -111,7 +111,7 @@ func (cgc *containerGC) removeOldestN(containers []containerGCInfo, toRemove int // Remove from oldest to newest (last to first). numToKeep := len(containers) - toRemove for i := numToKeep; i < len(containers); i++ { - err := cgc.client.RemoveContainer(docker.RemoveContainerOptions{ID: containers[i].id, RemoveVolumes: true}) + err := cgc.client.RemoveContainer(containers[i].id, dockertypes.ContainerRemoveOptions{RemoveVolumes: true}) if err != nil { glog.Warningf("Failed to remove dead container %q: %v", containers[i].name, err) } @@ -145,14 +145,20 @@ func (cgc *containerGC) evictableContainers(minAge time.Duration) (containersByE continue } else if data.State.Running { continue - } else if newestGCTime.Before(data.Created) { + } + + created, err := parseDockerTimestamp(data.Created) + if err != nil { + glog.Errorf("Failed to parse Created timestamp %q for container %q", data.Created, container.ID) + } + if newestGCTime.Before(created) { continue } containerInfo := containerGCInfo{ id: container.ID, name: container.Names[0], - createTime: data.Created, + createTime: created, } containerName, _, err := ParseDockerName(container.Names[0]) @@ -189,7 +195,7 @@ func (cgc *containerGC) GarbageCollect(gcPolicy kubecontainer.ContainerGCPolicy) // Remove unidentified containers. for _, container := range unidentifiedContainers { glog.Infof("Removing unidentified dead container %q with ID %q", container.name, container.id) - err = cgc.client.RemoveContainer(docker.RemoveContainerOptions{ID: container.id, RemoveVolumes: true}) + err = cgc.client.RemoveContainer(container.id, dockertypes.ContainerRemoveOptions{RemoveVolumes: true}) if err != nil { glog.Warningf("Failed to remove unidentified dead container %q: %v", container.name, err) } diff --git a/pkg/kubelet/dockertools/container_gc_test.go b/pkg/kubelet/dockertools/container_gc_test.go index d6038e26a05..36630ebed34 100644 --- a/pkg/kubelet/dockertools/container_gc_test.go +++ b/pkg/kubelet/dockertools/container_gc_test.go @@ -23,7 +23,6 @@ import ( "testing" "time" - docker "github.com/fsouza/go-dockerclient" "github.com/stretchr/testify/assert" "k8s.io/kubernetes/pkg/api" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -44,26 +43,22 @@ func makeTime(id int) time.Time { } // Makes a container with the specified properties. -func makeContainer(id, uid, name string, running bool, created time.Time) *docker.Container { - return &docker.Container{ - Name: fmt.Sprintf("/k8s_%s_bar_new_%s_42", name, uid), - State: docker.State{ - Running: running, - }, - ID: id, - Created: created, +func makeContainer(id, uid, name string, running bool, created time.Time) *FakeContainer { + return &FakeContainer{ + Name: fmt.Sprintf("/k8s_%s_bar_new_%s_42", name, uid), + Running: running, + ID: id, + CreatedAt: created, } } // Makes a container with unidentified name and specified properties. -func makeUndefinedContainer(id string, running bool, created time.Time) *docker.Container { - return &docker.Container{ - Name: "/k8s_unidentified", - State: docker.State{ - Running: running, - }, - ID: id, - Created: created, +func makeUndefinedContainer(id string, running bool, created time.Time) *FakeContainer { + return &FakeContainer{ + Name: "/k8s_unidentified", + Running: running, + ID: id, + CreatedAt: created, } } @@ -96,7 +91,7 @@ func verifyStringArrayEqualsAnyOrder(t *testing.T, actual, expected []string) { func TestGarbageCollectZeroMaxContainers(t *testing.T) { gc, fakeDocker := newTestContainerGC(t) - fakeDocker.SetFakeContainers([]*docker.Container{ + fakeDocker.SetFakeContainers([]*FakeContainer{ makeContainer("1876", "foo", "POD", false, makeTime(0)), }) addPods(gc.podGetter, "foo") @@ -107,7 +102,7 @@ func TestGarbageCollectZeroMaxContainers(t *testing.T) { func TestGarbageCollectNoMaxPerPodContainerLimit(t *testing.T) { gc, fakeDocker := newTestContainerGC(t) - fakeDocker.SetFakeContainers([]*docker.Container{ + fakeDocker.SetFakeContainers([]*FakeContainer{ makeContainer("1876", "foo", "POD", false, makeTime(0)), makeContainer("2876", "foo1", "POD", false, makeTime(1)), makeContainer("3876", "foo2", "POD", false, makeTime(2)), @@ -122,7 +117,7 @@ func TestGarbageCollectNoMaxPerPodContainerLimit(t *testing.T) { func TestGarbageCollectNoMaxLimit(t *testing.T) { gc, fakeDocker := newTestContainerGC(t) - fakeDocker.SetFakeContainers([]*docker.Container{ + fakeDocker.SetFakeContainers([]*FakeContainer{ makeContainer("1876", "foo", "POD", false, makeTime(0)), makeContainer("2876", "foo1", "POD", false, makeTime(0)), makeContainer("3876", "foo2", "POD", false, makeTime(0)), @@ -136,12 +131,12 @@ func TestGarbageCollectNoMaxLimit(t *testing.T) { func TestGarbageCollect(t *testing.T) { tests := []struct { - containers []*docker.Container + containers []*FakeContainer expectedRemoved []string }{ // Don't remove containers started recently. { - containers: []*docker.Container{ + containers: []*FakeContainer{ makeContainer("1876", "foo", "POD", false, time.Now()), makeContainer("2876", "foo", "POD", false, time.Now()), makeContainer("3876", "foo", "POD", false, time.Now()), @@ -149,7 +144,7 @@ func TestGarbageCollect(t *testing.T) { }, // Remove oldest containers. { - containers: []*docker.Container{ + containers: []*FakeContainer{ makeContainer("1876", "foo", "POD", false, makeTime(0)), makeContainer("2876", "foo", "POD", false, makeTime(1)), makeContainer("3876", "foo", "POD", false, makeTime(2)), @@ -158,7 +153,7 @@ func TestGarbageCollect(t *testing.T) { }, // Only remove non-running containers. { - containers: []*docker.Container{ + containers: []*FakeContainer{ makeContainer("1876", "foo", "POD", true, makeTime(0)), makeContainer("2876", "foo", "POD", false, makeTime(1)), makeContainer("3876", "foo", "POD", false, makeTime(2)), @@ -168,13 +163,13 @@ func TestGarbageCollect(t *testing.T) { }, // Less than maxContainerCount doesn't delete any. { - containers: []*docker.Container{ + containers: []*FakeContainer{ makeContainer("1876", "foo", "POD", false, makeTime(0)), }, }, // maxContainerCount applies per (UID,container) pair. { - containers: []*docker.Container{ + containers: []*FakeContainer{ makeContainer("1876", "foo", "POD", false, makeTime(0)), makeContainer("2876", "foo", "POD", false, makeTime(1)), makeContainer("3876", "foo", "POD", false, makeTime(2)), @@ -189,7 +184,7 @@ func TestGarbageCollect(t *testing.T) { }, // Remove non-running unidentified Kubernetes containers. { - containers: []*docker.Container{ + containers: []*FakeContainer{ makeUndefinedContainer("1876", true, makeTime(0)), makeUndefinedContainer("2876", false, makeTime(0)), makeContainer("3876", "foo", "POD", false, makeTime(0)), @@ -198,7 +193,7 @@ func TestGarbageCollect(t *testing.T) { }, // Max limit applied and tries to keep from every pod. { - containers: []*docker.Container{ + containers: []*FakeContainer{ makeContainer("1876", "foo", "POD", false, makeTime(0)), makeContainer("2876", "foo", "POD", false, makeTime(1)), makeContainer("3876", "foo1", "POD", false, makeTime(0)), @@ -214,7 +209,7 @@ func TestGarbageCollect(t *testing.T) { }, // If more pods than limit allows, evicts oldest pod. { - containers: []*docker.Container{ + containers: []*FakeContainer{ makeContainer("1876", "foo", "POD", false, makeTime(1)), makeContainer("2876", "foo", "POD", false, makeTime(2)), makeContainer("3876", "foo1", "POD", false, makeTime(1)), @@ -230,7 +225,7 @@ func TestGarbageCollect(t *testing.T) { }, // Containers for deleted pods should be GC'd. { - containers: []*docker.Container{ + containers: []*FakeContainer{ makeContainer("1876", "foo", "POD", false, makeTime(1)), makeContainer("2876", "foo", "POD", false, makeTime(2)), makeContainer("3876", "deleted", "POD", false, makeTime(1)), diff --git a/pkg/kubelet/dockertools/convert.go b/pkg/kubelet/dockertools/convert.go index 5de70028349..71c0c7551f8 100644 --- a/pkg/kubelet/dockertools/convert.go +++ b/pkg/kubelet/dockertools/convert.go @@ -20,6 +20,7 @@ import ( "fmt" "strings" + dockertypes "github.com/docker/engine-api/types" docker "github.com/fsouza/go-dockerclient" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) @@ -32,7 +33,7 @@ const ( ) func mapState(state string) kubecontainer.ContainerState { - // Parse the state string in docker.APIContainers. This could break when + // Parse the state string in dockertypes.Container. This could break when // we upgrade docker. switch { case strings.HasPrefix(state, statusRunningPrefix): @@ -44,8 +45,8 @@ func mapState(state string) kubecontainer.ContainerState { } } -// Converts docker.APIContainers to kubecontainer.Container. -func toRuntimeContainer(c *docker.APIContainers) (*kubecontainer.Container, error) { +// Converts dockertypes.Container to kubecontainer.Container. +func toRuntimeContainer(c *dockertypes.Container) (*kubecontainer.Container, error) { if c == nil { return nil, fmt.Errorf("unable to convert a nil pointer to a runtime container") } diff --git a/pkg/kubelet/dockertools/convert_test.go b/pkg/kubelet/dockertools/convert_test.go index 2e18ba77a8c..a8265395f9c 100644 --- a/pkg/kubelet/dockertools/convert_test.go +++ b/pkg/kubelet/dockertools/convert_test.go @@ -20,6 +20,7 @@ import ( "reflect" "testing" + dockertypes "github.com/docker/engine-api/types" docker "github.com/fsouza/go-dockerclient" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) @@ -43,7 +44,7 @@ func TestMapState(t *testing.T) { } func TestToRuntimeContainer(t *testing.T) { - original := &docker.APIContainers{ + original := &dockertypes.Container{ ID: "ab2cdf", Image: "bar_image", Created: 12345, diff --git a/pkg/kubelet/dockertools/docker.go b/pkg/kubelet/dockertools/docker.go index 1a59db28400..58f45dbe2b8 100644 --- a/pkg/kubelet/dockertools/docker.go +++ b/pkg/kubelet/dockertools/docker.go @@ -26,6 +26,7 @@ import ( "github.com/docker/docker/pkg/jsonmessage" dockerapi "github.com/docker/engine-api/client" + dockertypes "github.com/docker/engine-api/types" docker "github.com/fsouza/go-dockerclient" "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" @@ -57,12 +58,12 @@ const ( // DockerInterface is an abstract interface for testability. It abstracts the interface of docker.Client. type DockerInterface interface { - ListContainers(options docker.ListContainersOptions) ([]docker.APIContainers, error) - InspectContainer(id string) (*docker.Container, error) - CreateContainer(docker.CreateContainerOptions) (*docker.Container, error) - StartContainer(id string, hostConfig *docker.HostConfig) error - StopContainer(id string, timeout uint) error - RemoveContainer(opts docker.RemoveContainerOptions) error + ListContainers(options dockertypes.ContainerListOptions) ([]dockertypes.Container, error) + InspectContainer(id string) (*dockertypes.ContainerJSON, error) + CreateContainer(dockertypes.ContainerCreateConfig) (*dockertypes.ContainerCreateResponse, error) + StartContainer(id string) error + StopContainer(id string, timeout int) error + RemoveContainer(id string, opts dockertypes.ContainerRemoveOptions) error InspectImage(image string) (*docker.Image, error) ListImages(opts docker.ListImagesOptions) ([]docker.APIImages, error) PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) error @@ -346,9 +347,9 @@ func milliCPUToShares(milliCPU int64) int64 { // GetKubeletDockerContainers lists all container or just the running ones. // Returns a list of docker containers that we manage // TODO: Move this function with dockerCache to DockerManager. -func GetKubeletDockerContainers(client DockerInterface, allContainers bool) ([]*docker.APIContainers, error) { - result := []*docker.APIContainers{} - containers, err := client.ListContainers(docker.ListContainersOptions{All: allContainers}) +func GetKubeletDockerContainers(client DockerInterface, allContainers bool) ([]*dockertypes.Container, error) { + result := []*dockertypes.Container{} + containers, err := client.ListContainers(dockertypes.ContainerListOptions{All: allContainers}) if err != nil { return nil, err } diff --git a/pkg/kubelet/dockertools/docker_test.go b/pkg/kubelet/dockertools/docker_test.go index 555916ca5e9..c00a5589b01 100644 --- a/pkg/kubelet/dockertools/docker_test.go +++ b/pkg/kubelet/dockertools/docker_test.go @@ -27,6 +27,8 @@ import ( "testing" "github.com/docker/docker/pkg/jsonmessage" + dockertypes "github.com/docker/engine-api/types" + dockernat "github.com/docker/go-connections/nat" docker "github.com/fsouza/go-dockerclient" cadvisorapi "github.com/google/cadvisor/info/v1" "k8s.io/kubernetes/cmd/kubelet/app/options" @@ -62,7 +64,7 @@ func verifyStringArrayEquals(t *testing.T, actual, expected []string) { } } -func findPodContainer(dockerContainers []*docker.APIContainers, podFullName string, uid types.UID, containerName string) (*docker.APIContainers, bool, uint64) { +func findPodContainer(dockerContainers []*dockertypes.Container, podFullName string, uid types.UID, containerName string) (*dockertypes.Container, bool, uint64) { for _, dockerContainer := range dockerContainers { if len(dockerContainer.Names) == 0 { continue @@ -82,7 +84,7 @@ func findPodContainer(dockerContainers []*docker.APIContainers, podFullName stri func TestGetContainerID(t *testing.T) { fakeDocker := NewFakeDockerClient() - fakeDocker.SetFakeRunningContainers([]*docker.Container{ + fakeDocker.SetFakeRunningContainers([]*FakeContainer{ { ID: "foobar", Name: "/k8s_foo_qux_ns_1234_42", @@ -98,7 +100,7 @@ func TestGetContainerID(t *testing.T) { t.Errorf("Expected no error, Got %#v", err) } if len(dockerContainers) != 2 { - t.Errorf("Expected %#v, Got %#v", fakeDocker.ContainerList, dockerContainers) + t.Errorf("Expected %#v, Got %#v", fakeDocker.RunningContainerList, dockerContainers) } verifyCalls(t, fakeDocker, []string{"list"}) @@ -522,14 +524,14 @@ func (b containersByID) Less(i, j int) bool { return b[i].ID.ID < b[j].ID.ID } func TestFindContainersByPod(t *testing.T) { tests := []struct { - containerList []docker.APIContainers - exitedContainerList []docker.APIContainers - all bool - expectedPods []*kubecontainer.Pod + runningContainerList []dockertypes.Container + exitedContainerList []dockertypes.Container + all bool + expectedPods []*kubecontainer.Pod }{ { - []docker.APIContainers{ + []dockertypes.Container{ { ID: "foobar", Names: []string{"/k8s_foobar.1234_qux_ns_1234_42"}, @@ -543,7 +545,7 @@ func TestFindContainersByPod(t *testing.T) { Names: []string{"/k8s_baz.1234_qux_ns_1234_42"}, }, }, - []docker.APIContainers{ + []dockertypes.Container{ { ID: "barfoo", Names: []string{"/k8s_barfoo.1234_qux_ns_1234_42"}, @@ -590,7 +592,7 @@ func TestFindContainersByPod(t *testing.T) { }, }, { - []docker.APIContainers{ + []dockertypes.Container{ { ID: "foobar", Names: []string{"/k8s_foobar.1234_qux_ns_1234_42"}, @@ -604,7 +606,7 @@ func TestFindContainersByPod(t *testing.T) { Names: []string{"/k8s_baz.1234_qux_ns_1234_42"}, }, }, - []docker.APIContainers{ + []dockertypes.Container{ { ID: "barfoo", Names: []string{"/k8s_barfoo.1234_qux_ns_1234_42"}, @@ -670,8 +672,8 @@ func TestFindContainersByPod(t *testing.T) { }, }, { - []docker.APIContainers{}, - []docker.APIContainers{}, + []dockertypes.Container{}, + []dockertypes.Container{}, true, nil, }, @@ -681,7 +683,7 @@ func TestFindContainersByPod(t *testing.T) { // image back-off is set to nil, this test should not pull images containerManager := NewFakeDockerManager(fakeClient, &record.FakeRecorder{}, nil, nil, &cadvisorapi.MachineInfo{}, options.GetDefaultPodInfraContainerImage(), 0, 0, "", containertest.FakeOS{}, np, nil, nil, nil) for i, test := range tests { - fakeClient.ContainerList = test.containerList + fakeClient.RunningContainerList = test.runningContainerList fakeClient.ExitedContainerList = test.exitedContainerList result, _ := containerManager.GetPods(test.all) @@ -749,37 +751,37 @@ func TestMakePortsAndBindings(t *testing.T) { } // Construct expected bindings - expectPortBindings := map[string][]docker.PortBinding{ + expectPortBindings := map[string][]dockernat.PortBinding{ "80/tcp": { - docker.PortBinding{ + dockernat.PortBinding{ HostPort: "8080", HostIP: "127.0.0.1", }, }, "443/tcp": { - docker.PortBinding{ + dockernat.PortBinding{ HostPort: "443", HostIP: "", }, - docker.PortBinding{ + dockernat.PortBinding{ HostPort: "446", HostIP: "", }, }, "443/udp": { - docker.PortBinding{ + dockernat.PortBinding{ HostPort: "446", HostIP: "", }, }, "444/udp": { - docker.PortBinding{ + dockernat.PortBinding{ HostPort: "444", HostIP: "", }, }, "445/tcp": { - docker.PortBinding{ + dockernat.PortBinding{ HostPort: "445", HostIP: "", }, diff --git a/pkg/kubelet/dockertools/exec.go b/pkg/kubelet/dockertools/exec.go index 426b5fb2e03..9b82977d232 100644 --- a/pkg/kubelet/dockertools/exec.go +++ b/pkg/kubelet/dockertools/exec.go @@ -23,6 +23,7 @@ import ( "os/exec" "time" + dockertypes "github.com/docker/engine-api/types" docker "github.com/fsouza/go-dockerclient" "github.com/golang/glog" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -30,14 +31,14 @@ import ( // ExecHandler knows how to execute a command in a running Docker container. type ExecHandler interface { - ExecInContainer(client DockerInterface, container *docker.Container, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool) error + ExecInContainer(client DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool) error } // NsenterExecHandler executes commands in Docker containers using nsenter. type NsenterExecHandler struct{} // TODO should we support nsenter in a container, running with elevated privs and --pid=host? -func (*NsenterExecHandler) ExecInContainer(client DockerInterface, container *docker.Container, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool) error { +func (*NsenterExecHandler) ExecInContainer(client DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool) error { nsenter, err := exec.LookPath("nsenter") if err != nil { return fmt.Errorf("exec unavailable - unable to locate nsenter") @@ -98,7 +99,7 @@ func (*NsenterExecHandler) ExecInContainer(client DockerInterface, container *do // NativeExecHandler executes commands in Docker containers using Docker's exec API. type NativeExecHandler struct{} -func (*NativeExecHandler) ExecInContainer(client DockerInterface, container *docker.Container, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool) error { +func (*NativeExecHandler) ExecInContainer(client DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool) error { createOpts := docker.CreateExecOptions{ Container: container.ID, Cmd: cmd, diff --git a/pkg/kubelet/dockertools/fake_docker_client.go b/pkg/kubelet/dockertools/fake_docker_client.go index 81037152e28..eb52bfcd162 100644 --- a/pkg/kubelet/dockertools/fake_docker_client.go +++ b/pkg/kubelet/dockertools/fake_docker_client.go @@ -26,6 +26,8 @@ import ( "sync" "time" + dockertypes "github.com/docker/engine-api/types" + dockercontainer "github.com/docker/engine-api/types/container" docker "github.com/fsouza/go-dockerclient" "k8s.io/kubernetes/pkg/api" @@ -35,14 +37,14 @@ import ( // FakeDockerClient is a simple fake docker client, so that kubelet can be run for testing without requiring a real docker setup. type FakeDockerClient struct { sync.Mutex - ContainerList []docker.APIContainers - ExitedContainerList []docker.APIContainers - ContainerMap map[string]*docker.Container - Image *docker.Image - Images []docker.APIImages - Errors map[string]error - called []string - pulled []string + RunningContainerList []dockertypes.Container + ExitedContainerList []dockertypes.Container + ContainerMap map[string]*dockertypes.ContainerJSON + Image *docker.Image + Images []docker.APIImages + Errors map[string]error + called []string + pulled []string // Created, Stopped and Removed all container docker ID Created []string Stopped []string @@ -68,7 +70,7 @@ func NewFakeDockerClientWithVersion(version, apiVersion string) *FakeDockerClien VersionInfo: docker.Env{fmt.Sprintf("Version=%s", version), fmt.Sprintf("ApiVersion=%s", apiVersion)}, Errors: make(map[string]error), RemovedImages: sets.String{}, - ContainerMap: make(map[string]*docker.Container), + ContainerMap: make(map[string]*dockertypes.ContainerJSON), } } @@ -102,35 +104,74 @@ func (f *FakeDockerClient) ClearCalls() { f.Removed = []string{} } -func (f *FakeDockerClient) SetFakeContainers(containers []*docker.Container) { +// Because the new data type returned by engine-api is too complex to manually initialize, we need a +// fake container which is easier to initialize. +type FakeContainer struct { + ID string + Name string + Running bool + ExitCode int + Pid int + CreatedAt time.Time + StartedAt time.Time + FinishedAt time.Time + Config *dockercontainer.Config + HostConfig *dockercontainer.HostConfig +} + +// convertFakeContainer converts the fake container to real container +func convertFakeContainer(f *FakeContainer) *dockertypes.ContainerJSON { + if f.Config == nil { + f.Config = &dockercontainer.Config{} + } + if f.HostConfig == nil { + f.HostConfig = &dockercontainer.HostConfig{} + } + return &dockertypes.ContainerJSON{ + ContainerJSONBase: &dockertypes.ContainerJSONBase{ + ID: f.ID, + Name: f.Name, + State: &dockertypes.ContainerState{ + Running: f.Running, + ExitCode: f.ExitCode, + Pid: f.Pid, + StartedAt: dockerTimestampToString(f.StartedAt), + FinishedAt: dockerTimestampToString(f.FinishedAt), + }, + Created: dockerTimestampToString(f.CreatedAt), + HostConfig: f.HostConfig, + }, + Config: f.Config, + NetworkSettings: &dockertypes.NetworkSettings{}, + } +} + +func (f *FakeDockerClient) SetFakeContainers(containers []*FakeContainer) { f.Lock() defer f.Unlock() // Reset the lists and the map. - f.ContainerMap = map[string]*docker.Container{} - f.ContainerList = []docker.APIContainers{} - f.ExitedContainerList = []docker.APIContainers{} + f.ContainerMap = map[string]*dockertypes.ContainerJSON{} + f.RunningContainerList = []dockertypes.Container{} + f.ExitedContainerList = []dockertypes.Container{} for i := range containers { c := containers[i] - if c.Config == nil { - c.Config = &docker.Config{} - } - f.ContainerMap[c.ID] = c - apiContainer := docker.APIContainers{ + f.ContainerMap[c.ID] = convertFakeContainer(c) + container := dockertypes.Container{ Names: []string{c.Name}, ID: c.ID, } - if c.State.Running { - f.ContainerList = append(f.ContainerList, apiContainer) + if c.Running { + f.RunningContainerList = append(f.RunningContainerList, container) } else { - f.ExitedContainerList = append(f.ExitedContainerList, apiContainer) + f.ExitedContainerList = append(f.ExitedContainerList, container) } } } -func (f *FakeDockerClient) SetFakeRunningContainers(containers []*docker.Container) { +func (f *FakeDockerClient) SetFakeRunningContainers(containers []*FakeContainer) { for _, c := range containers { - c.State.Running = true + c.Running = true } f.SetFakeContainers(containers) } @@ -210,12 +251,12 @@ func (f *FakeDockerClient) popError(op string) error { // ListContainers is a test-spy implementation of DockerInterface.ListContainers. // It adds an entry "list" to the internal method call record. -func (f *FakeDockerClient) ListContainers(options docker.ListContainersOptions) ([]docker.APIContainers, error) { +func (f *FakeDockerClient) ListContainers(options dockertypes.ContainerListOptions) ([]dockertypes.Container, error) { f.Lock() defer f.Unlock() f.called = append(f.called, "list") err := f.popError("list") - containerList := append([]docker.APIContainers{}, f.ContainerList...) + containerList := append([]dockertypes.Container{}, f.RunningContainerList...) if options.All { // Although the container is not sorted, but the container with the same name should be in order, // that is enough for us now. @@ -227,7 +268,7 @@ func (f *FakeDockerClient) ListContainers(options docker.ListContainersOptions) // InspectContainer is a test-spy implementation of DockerInterface.InspectContainer. // It adds an entry "inspect" to the internal method call record. -func (f *FakeDockerClient) InspectContainer(id string) (*docker.Container, error) { +func (f *FakeDockerClient) InspectContainer(id string) (*dockertypes.ContainerJSON, error) { f.Lock() defer f.Unlock() f.called = append(f.called, "inspect_container") @@ -264,7 +305,7 @@ func (f *FakeDockerClient) normalSleep(mean, stdDev, cutOffMillis int) { // CreateContainer is a test-spy implementation of DockerInterface.CreateContainer. // It adds an entry "create" to the internal method call record. -func (f *FakeDockerClient) CreateContainer(c docker.CreateContainerOptions) (*docker.Container, error) { +func (f *FakeDockerClient) CreateContainer(c dockertypes.ContainerCreateConfig) (*dockertypes.ContainerCreateResponse, error) { f.Lock() defer f.Unlock() f.called = append(f.called, "create") @@ -274,25 +315,20 @@ func (f *FakeDockerClient) CreateContainer(c docker.CreateContainerOptions) (*do // This is not a very good fake. We'll just add this container's name to the list. // Docker likes to add a '/', so copy that behavior. name := "/" + c.Name + id := name f.Created = append(f.Created, name) // The newest container should be in front, because we assume so in GetPodStatus() - f.ContainerList = append([]docker.APIContainers{ + f.RunningContainerList = append([]dockertypes.Container{ {ID: name, Names: []string{name}, Image: c.Config.Image, Labels: c.Config.Labels}, - }, f.ContainerList...) - container := docker.Container{ID: name, Name: name, Config: c.Config, HostConfig: c.HostConfig} - containerCopy := container - f.ContainerMap[name] = &containerCopy + }, f.RunningContainerList...) + f.ContainerMap[name] = convertFakeContainer(&FakeContainer{ID: id, Name: name, Config: c.Config, HostConfig: c.HostConfig}) f.normalSleep(100, 25, 25) - return &container, nil + return &dockertypes.ContainerCreateResponse{ID: id}, nil } // StartContainer is a test-spy implementation of DockerInterface.StartContainer. // It adds an entry "start" to the internal method call record. -// The HostConfig at StartContainer will be deprecated from docker 1.10. Now in -// docker manager the HostConfig is set when CreateContainer(). -// TODO(random-liu): Remove the HostConfig here when it is completely removed in -// docker 1.12. -func (f *FakeDockerClient) StartContainer(id string, _ *docker.HostConfig) error { +func (f *FakeDockerClient) StartContainer(id string) error { f.Lock() defer f.Unlock() f.called = append(f.called, "start") @@ -301,14 +337,12 @@ func (f *FakeDockerClient) StartContainer(id string, _ *docker.HostConfig) error } container, ok := f.ContainerMap[id] if !ok { - container = &docker.Container{ID: id, Name: id} + container = convertFakeContainer(&FakeContainer{ID: id, Name: id}) } - container.State = docker.State{ - Running: true, - Pid: os.Getpid(), - StartedAt: time.Now(), - } - container.NetworkSettings = &docker.NetworkSettings{IPAddress: "2.3.4.5"} + container.State.Running = true + container.State.Pid = os.Getpid() + container.State.StartedAt = dockerTimestampToString(time.Now()) + container.NetworkSettings.IPAddress = "2.3.4.5" f.ContainerMap[id] = container f.updateContainerStatus(id, statusRunningPrefix) f.normalSleep(200, 50, 50) @@ -317,7 +351,7 @@ func (f *FakeDockerClient) StartContainer(id string, _ *docker.HostConfig) error // StopContainer is a test-spy implementation of DockerInterface.StopContainer. // It adds an entry "stop" to the internal method call record. -func (f *FakeDockerClient) StopContainer(id string, timeout uint) error { +func (f *FakeDockerClient) StopContainer(id string, timeout int) error { f.Lock() defer f.Unlock() f.called = append(f.called, "stop") @@ -327,29 +361,27 @@ func (f *FakeDockerClient) StopContainer(id string, timeout uint) error { f.Stopped = append(f.Stopped, id) // Container status should be Updated before container moved to ExitedContainerList f.updateContainerStatus(id, statusExitedPrefix) - var newList []docker.APIContainers - for _, container := range f.ContainerList { + var newList []dockertypes.Container + for _, container := range f.RunningContainerList { if container.ID == id { // The newest exited container should be in front. Because we assume so in GetPodStatus() - f.ExitedContainerList = append([]docker.APIContainers{container}, f.ExitedContainerList...) + f.ExitedContainerList = append([]dockertypes.Container{container}, f.ExitedContainerList...) continue } newList = append(newList, container) } - f.ContainerList = newList + f.RunningContainerList = newList container, ok := f.ContainerMap[id] if !ok { - container = &docker.Container{ - ID: id, - Name: id, - State: docker.State{ - Running: false, - StartedAt: time.Now().Add(-time.Second), - FinishedAt: time.Now(), - }, - } + container = convertFakeContainer(&FakeContainer{ + ID: id, + Name: id, + Running: false, + StartedAt: time.Now().Add(-time.Second), + FinishedAt: time.Now(), + }) } else { - container.State.FinishedAt = time.Now() + container.State.FinishedAt = dockerTimestampToString(time.Now()) container.State.Running = false } f.ContainerMap[id] = container @@ -357,7 +389,7 @@ func (f *FakeDockerClient) StopContainer(id string, timeout uint) error { return nil } -func (f *FakeDockerClient) RemoveContainer(opts docker.RemoveContainerOptions) error { +func (f *FakeDockerClient) RemoveContainer(id string, opts dockertypes.ContainerRemoveOptions) error { f.Lock() defer f.Unlock() f.called = append(f.called, "remove") @@ -366,10 +398,10 @@ func (f *FakeDockerClient) RemoveContainer(opts docker.RemoveContainerOptions) e return err } for i := range f.ExitedContainerList { - if f.ExitedContainerList[i].ID == opts.ID { - delete(f.ContainerMap, opts.ID) + if f.ExitedContainerList[i].ID == id { + delete(f.ContainerMap, id) f.ExitedContainerList = append(f.ExitedContainerList[:i], f.ExitedContainerList[i+1:]...) - f.Removed = append(f.Removed, opts.ID) + f.Removed = append(f.Removed, id) return nil } @@ -387,7 +419,7 @@ func (f *FakeDockerClient) Logs(opts docker.LogsOptions) error { return f.popError("logs") } -// PullImage is a test-spy implementation of DockerInterface.StopContainer. +// PullImage is a test-spy implementation of DockerInterface.PullImage. // It adds an entry "pull" to the internal method call record. func (f *FakeDockerClient) PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) error { f.Lock() @@ -455,9 +487,9 @@ func (f *FakeDockerClient) RemoveImage(image string) error { } func (f *FakeDockerClient) updateContainerStatus(id, status string) { - for i := range f.ContainerList { - if f.ContainerList[i].ID == id { - f.ContainerList[i].Status = status + for i := range f.RunningContainerList { + if f.RunningContainerList[i].ID == id { + f.RunningContainerList[i].Status = status } } } @@ -500,3 +532,8 @@ func (f *FakeDockerPuller) IsImagePresent(name string) (bool, error) { } return false, nil } + +// dockerTimestampToString converts the timestamp to string +func dockerTimestampToString(t time.Time) string { + return t.Format(time.RFC3339Nano) +} diff --git a/pkg/kubelet/dockertools/instrumented_docker.go b/pkg/kubelet/dockertools/instrumented_docker.go index e90c3f75440..c9053132c3f 100644 --- a/pkg/kubelet/dockertools/instrumented_docker.go +++ b/pkg/kubelet/dockertools/instrumented_docker.go @@ -19,6 +19,7 @@ package dockertools import ( "time" + dockertypes "github.com/docker/engine-api/types" docker "github.com/fsouza/go-dockerclient" "k8s.io/kubernetes/pkg/kubelet/metrics" ) @@ -48,7 +49,7 @@ func recordError(operation string, err error) { } } -func (in instrumentedDockerInterface) ListContainers(options docker.ListContainersOptions) ([]docker.APIContainers, error) { +func (in instrumentedDockerInterface) ListContainers(options dockertypes.ContainerListOptions) ([]dockertypes.Container, error) { const operation = "list_containers" defer recordOperation(operation, time.Now()) @@ -57,7 +58,7 @@ func (in instrumentedDockerInterface) ListContainers(options docker.ListContaine return out, err } -func (in instrumentedDockerInterface) InspectContainer(id string) (*docker.Container, error) { +func (in instrumentedDockerInterface) InspectContainer(id string) (*dockertypes.ContainerJSON, error) { const operation = "inspect_container" defer recordOperation(operation, time.Now()) @@ -66,7 +67,7 @@ func (in instrumentedDockerInterface) InspectContainer(id string) (*docker.Conta return out, err } -func (in instrumentedDockerInterface) CreateContainer(opts docker.CreateContainerOptions) (*docker.Container, error) { +func (in instrumentedDockerInterface) CreateContainer(opts dockertypes.ContainerCreateConfig) (*dockertypes.ContainerCreateResponse, error) { const operation = "create_container" defer recordOperation(operation, time.Now()) @@ -75,16 +76,16 @@ func (in instrumentedDockerInterface) CreateContainer(opts docker.CreateContaine return out, err } -func (in instrumentedDockerInterface) StartContainer(id string, hostConfig *docker.HostConfig) error { +func (in instrumentedDockerInterface) StartContainer(id string) error { const operation = "start_container" defer recordOperation(operation, time.Now()) - err := in.client.StartContainer(id, hostConfig) + err := in.client.StartContainer(id) recordError(operation, err) return err } -func (in instrumentedDockerInterface) StopContainer(id string, timeout uint) error { +func (in instrumentedDockerInterface) StopContainer(id string, timeout int) error { const operation = "stop_container" defer recordOperation(operation, time.Now()) @@ -93,11 +94,11 @@ func (in instrumentedDockerInterface) StopContainer(id string, timeout uint) err return err } -func (in instrumentedDockerInterface) RemoveContainer(opts docker.RemoveContainerOptions) error { +func (in instrumentedDockerInterface) RemoveContainer(id string, opts dockertypes.ContainerRemoveOptions) error { const operation = "remove_container" defer recordOperation(operation, time.Now()) - err := in.client.RemoveContainer(opts) + err := in.client.RemoveContainer(id, opts) recordError(operation, err) return err } diff --git a/pkg/kubelet/dockertools/kube_docker_client.go b/pkg/kubelet/dockertools/kube_docker_client.go index 599d5d0cef3..60f8dd08055 100644 --- a/pkg/kubelet/dockertools/kube_docker_client.go +++ b/pkg/kubelet/dockertools/kube_docker_client.go @@ -20,14 +20,15 @@ import ( "bytes" "encoding/base64" "encoding/json" + "fmt" "io" "io/ioutil" "strconv" + "time" "github.com/docker/docker/pkg/stdcopy" dockerapi "github.com/docker/engine-api/client" dockertypes "github.com/docker/engine-api/types" - dockercontainer "github.com/docker/engine-api/types/container" dockerfilters "github.com/docker/engine-api/types/filters" docker "github.com/fsouza/go-dockerclient" "golang.org/x/net/context" @@ -100,77 +101,49 @@ func convertEnv(src interface{}) (*docker.Env, error) { return env, nil } -func (k *kubeDockerClient) ListContainers(options docker.ListContainersOptions) ([]docker.APIContainers, error) { - containers, err := k.client.ContainerList(getDefaultContext(), dockertypes.ContainerListOptions{ - Size: options.Size, - All: options.All, - Limit: options.Limit, - Since: options.Since, - Before: options.Before, - Filter: convertFilters(options.Filters), - }) +func (k *kubeDockerClient) ListContainers(options dockertypes.ContainerListOptions) ([]dockertypes.Container, error) { + containers, err := k.client.ContainerList(getDefaultContext(), options) if err != nil { return nil, err } - apiContainers := []docker.APIContainers{} - if err := convertType(&containers, &apiContainers); err != nil { - return nil, err + apiContainers := []dockertypes.Container{} + for _, c := range containers { + apiContainers = append(apiContainers, dockertypes.Container(c)) } return apiContainers, nil } -func (d *kubeDockerClient) InspectContainer(id string) (*docker.Container, error) { +func (d *kubeDockerClient) InspectContainer(id string) (*dockertypes.ContainerJSON, error) { containerJSON, err := d.client.ContainerInspect(getDefaultContext(), id) if err != nil { - // TODO(random-liu): Use IsErrContainerNotFound instead of NoSuchContainer error if dockerapi.IsErrContainerNotFound(err) { - err = &docker.NoSuchContainer{ID: id, Err: err} + return nil, containerNotFoundError{ID: id} } return nil, err } - container := &docker.Container{} - if err := convertType(&containerJSON, container); err != nil { - return nil, err - } - return container, nil + return &containerJSON, nil } -func (d *kubeDockerClient) CreateContainer(opts docker.CreateContainerOptions) (*docker.Container, error) { - config := &dockercontainer.Config{} - if err := convertType(opts.Config, config); err != nil { - return nil, err - } - hostConfig := &dockercontainer.HostConfig{} - if err := convertType(opts.HostConfig, hostConfig); err != nil { - return nil, err - } - resp, err := d.client.ContainerCreate(getDefaultContext(), config, hostConfig, nil, opts.Name) +func (d *kubeDockerClient) CreateContainer(opts dockertypes.ContainerCreateConfig) (*dockertypes.ContainerCreateResponse, error) { + createResp, err := d.client.ContainerCreate(getDefaultContext(), opts.Config, opts.HostConfig, opts.NetworkingConfig, opts.Name) if err != nil { return nil, err } - container := &docker.Container{} - if err := convertType(&resp, container); err != nil { - return nil, err - } - return container, nil + return &createResp, nil } -// TODO(random-liu): The HostConfig at container start is deprecated, will remove this in the following refactoring. -func (d *kubeDockerClient) StartContainer(id string, _ *docker.HostConfig) error { +func (d *kubeDockerClient) StartContainer(id string) error { return d.client.ContainerStart(getDefaultContext(), id) } // Stopping an already stopped container will not cause an error in engine-api. -func (d *kubeDockerClient) StopContainer(id string, timeout uint) error { - return d.client.ContainerStop(getDefaultContext(), id, int(timeout)) +func (d *kubeDockerClient) StopContainer(id string, timeout int) error { + return d.client.ContainerStop(getDefaultContext(), id, timeout) } -func (d *kubeDockerClient) RemoveContainer(opts docker.RemoveContainerOptions) error { - return d.client.ContainerRemove(getDefaultContext(), dockertypes.ContainerRemoveOptions{ - ContainerID: opts.ID, - RemoveVolumes: opts.RemoveVolumes, - Force: opts.Force, - }) +func (d *kubeDockerClient) RemoveContainer(id string, opts dockertypes.ContainerRemoveOptions) error { + opts.ContainerID = id + return d.client.ContainerRemove(getDefaultContext(), opts) } func (d *kubeDockerClient) InspectImage(image string) (*docker.Image, error) { @@ -387,3 +360,20 @@ func (d *kubeDockerClient) holdHijackedConnection(tty bool, inputStream io.Reade } return nil } + +// parseDockerTimestamp parses the timestamp returned by DockerInterface from string to time.Time +func parseDockerTimestamp(s string) (time.Time, error) { + // Timestamp returned by Docker is in time.RFC3339Nano format. + return time.Parse(time.RFC3339Nano, s) +} + +// containerNotFoundError is the error returned by InspectContainer when container not found. We +// add this error type for testability. We don't use the original error returned by engine-api +// because dockertypes.containerNotFoundError is private, we can't create and inject it in our test. +type containerNotFoundError struct { + ID string +} + +func (e containerNotFoundError) Error() string { + return fmt.Sprintf("Error: No such container: %s", e.ID) +} diff --git a/pkg/kubelet/dockertools/manager.go b/pkg/kubelet/dockertools/manager.go index b40e6af0158..cb555102fcb 100644 --- a/pkg/kubelet/dockertools/manager.go +++ b/pkg/kubelet/dockertools/manager.go @@ -32,6 +32,10 @@ import ( "time" "github.com/coreos/go-semver/semver" + dockertypes "github.com/docker/engine-api/types" + dockercontainer "github.com/docker/engine-api/types/container" + dockerstrslice "github.com/docker/engine-api/types/strslice" + dockernat "github.com/docker/go-connections/nat" docker "github.com/fsouza/go-dockerclient" "github.com/golang/glog" cadvisorapi "github.com/google/cadvisor/info/v1" @@ -313,7 +317,7 @@ var ( // determineContainerIP determines the IP address of the given container. It is expected // that the container passed is the infrastructure container of a pod and the responsibility // of the caller to ensure that the correct container is passed. -func (dm *DockerManager) determineContainerIP(podNamespace, podName string, container *docker.Container) string { +func (dm *DockerManager) determineContainerIP(podNamespace, podName string, container *dockertypes.ContainerJSON) string { result := "" if container.NetworkSettings != nil { @@ -351,6 +355,20 @@ func (dm *DockerManager) inspectContainer(id string, podName, podNamespace strin var containerInfo *labelledContainerInfo containerInfo = getContainerInfoFromLabel(iResult.Config.Labels) + parseTimestampError := func(label, s string) { + glog.Errorf("Failed to parse %q timestamp %q for container %q of pod %q", label, s, id, kubecontainer.BuildPodFullName(podName, podNamespace)) + } + var createdAt, startedAt, finishedAt time.Time + if createdAt, err = parseDockerTimestamp(iResult.Created); err != nil { + parseTimestampError("Created", iResult.Created) + } + if startedAt, err = parseDockerTimestamp(iResult.State.StartedAt); err != nil { + parseTimestampError("StartedAt", iResult.State.StartedAt) + } + if finishedAt, err = parseDockerTimestamp(iResult.State.FinishedAt); err != nil { + parseTimestampError("FinishedAt", iResult.State.FinishedAt) + } + status := kubecontainer.ContainerStatus{ Name: containerName, RestartCount: containerInfo.RestartCount, @@ -358,13 +376,13 @@ func (dm *DockerManager) inspectContainer(id string, podName, podNamespace strin ImageID: DockerPrefix + iResult.Image, ID: kubecontainer.DockerID(id).ContainerID(), ExitCode: iResult.State.ExitCode, - CreatedAt: iResult.Created, + CreatedAt: createdAt, Hash: hash, } if iResult.State.Running { // Container that are running, restarting and paused status.State = kubecontainer.ContainerStateRunning - status.StartedAt = iResult.State.StartedAt + status.StartedAt = startedAt if containerName == PodInfraContainerName { ip = dm.determineContainerIP(podNamespace, podName, iResult) } @@ -372,13 +390,11 @@ func (dm *DockerManager) inspectContainer(id string, podName, podNamespace strin } // Find containers that have exited or failed to start. - if !iResult.State.FinishedAt.IsZero() || iResult.State.ExitCode != 0 { + if !finishedAt.IsZero() || iResult.State.ExitCode != 0 { // Containers that are exited, dead or created (docker failed to start container) // When a container fails to start State.ExitCode is non-zero, FinishedAt and StartedAt are both zero reason := "" message := iResult.State.Error - finishedAt := iResult.State.FinishedAt - startedAt := iResult.State.StartedAt // Note: An application might handle OOMKilled gracefully. // In that case, the container is oom killed, but the exit @@ -387,14 +403,14 @@ func (dm *DockerManager) inspectContainer(id string, podName, podNamespace strin reason = "OOMKilled" } else if iResult.State.ExitCode == 0 { reason = "Completed" - } else if !iResult.State.FinishedAt.IsZero() { + } else if !finishedAt.IsZero() { reason = "Error" } else { // finishedAt is zero and ExitCode is nonZero occurs when docker fails to start the container reason = ErrContainerCannotRun.Error() // Adjust time to the time docker attempted to run the container, otherwise startedAt and finishedAt will be set to epoch, which is misleading - finishedAt = iResult.Created - startedAt = iResult.Created + finishedAt = createdAt + startedAt = createdAt } terminationMessagePath := containerInfo.TerminationMessagePath @@ -464,9 +480,9 @@ func makeMountBindings(mounts []kubecontainer.Mount, podHasSELinuxLabel bool) (r return } -func makePortsAndBindings(portMappings []kubecontainer.PortMapping) (map[docker.Port]struct{}, map[docker.Port][]docker.PortBinding) { - exposedPorts := map[docker.Port]struct{}{} - portBindings := map[docker.Port][]docker.PortBinding{} +func makePortsAndBindings(portMappings []kubecontainer.PortMapping) (map[dockernat.Port]struct{}, map[dockernat.Port][]dockernat.PortBinding) { + exposedPorts := map[dockernat.Port]struct{}{} + portBindings := map[dockernat.Port][]dockernat.PortBinding{} for _, port := range portMappings { exteriorPort := port.HostPort if exteriorPort == 0 { @@ -487,10 +503,10 @@ func makePortsAndBindings(portMappings []kubecontainer.PortMapping) (map[docker. protocol = "/tcp" } - dockerPort := docker.Port(strconv.Itoa(interiorPort) + protocol) + dockerPort := dockernat.Port(strconv.Itoa(interiorPort) + protocol) exposedPorts[dockerPort] = struct{}{} - hostBinding := docker.PortBinding{ + hostBinding := dockernat.PortBinding{ HostPort: strconv.Itoa(exteriorPort), HostIP: port.HostIP, } @@ -501,7 +517,7 @@ func makePortsAndBindings(portMappings []kubecontainer.PortMapping) (map[docker. portBindings[dockerPort] = append(existedBindings, hostBinding) } else { // Otherwise, it's fresh new port binding - portBindings[dockerPort] = []docker.PortBinding{ + portBindings[dockerPort] = []dockernat.PortBinding{ hostBinding, } } @@ -586,17 +602,18 @@ func (dm *DockerManager) runContainer( } } - hc := &docker.HostConfig{ + hc := &dockercontainer.HostConfig{ Binds: binds, - NetworkMode: netMode, - IpcMode: ipcMode, - UTSMode: utsMode, - PidMode: pidMode, + NetworkMode: dockercontainer.NetworkMode(netMode), + IpcMode: dockercontainer.IpcMode(ipcMode), + UTSMode: dockercontainer.UTSMode(utsMode), + PidMode: dockercontainer.PidMode(pidMode), ReadonlyRootfs: readOnlyRootFilesystem(container), - // Memory and CPU are set here for newer versions of Docker (1.6+). - Memory: memoryLimit, - MemorySwap: -1, - CPUShares: cpuShares, + Resources: dockercontainer.Resources{ + Memory: memoryLimit, + MemorySwap: -1, + CPUShares: cpuShares, + }, SecurityOpt: securityOpts, } @@ -620,15 +637,11 @@ func (dm *DockerManager) runContainer( hc.CgroupParent = opts.CgroupParent } - dockerOpts := docker.CreateContainerOptions{ + dockerOpts := dockertypes.ContainerCreateConfig{ Name: containerName, - Config: &docker.Config{ - Env: makeEnvList(opts.Envs), - Image: container.Image, - // Memory and CPU are set here for older versions of Docker (pre-1.6). - Memory: memoryLimit, - MemorySwap: -1, - CPUShares: cpuShares, + Config: &dockercontainer.Config{ + Env: makeEnvList(opts.Envs), + Image: container.Image, WorkingDir: container.WorkingDir, Labels: labels, // Interactive containers: @@ -644,36 +657,39 @@ func (dm *DockerManager) runContainer( setInfraContainerNetworkConfig(pod, netMode, opts, dockerOpts) } - setEntrypointAndCommand(container, opts, &dockerOpts) + setEntrypointAndCommand(container, opts, dockerOpts) glog.V(3).Infof("Container %v/%v/%v: setting entrypoint \"%v\" and command \"%v\"", pod.Namespace, pod.Name, container.Name, dockerOpts.Config.Entrypoint, dockerOpts.Config.Cmd) securityContextProvider := securitycontext.NewSimpleSecurityContextProvider() securityContextProvider.ModifyContainerConfig(pod, container, dockerOpts.Config) securityContextProvider.ModifyHostConfig(pod, container, dockerOpts.HostConfig) - dockerContainer, err := dm.client.CreateContainer(dockerOpts) + createResp, err := dm.client.CreateContainer(dockerOpts) if err != nil { dm.recorder.Eventf(ref, api.EventTypeWarning, kubecontainer.FailedToCreateContainer, "Failed to create docker container with error: %v", err) return kubecontainer.ContainerID{}, err } - dm.recorder.Eventf(ref, api.EventTypeNormal, kubecontainer.CreatedContainer, "Created container with docker id %v", utilstrings.ShortenString(dockerContainer.ID, 12)) + if len(createResp.Warnings) != 0 { + glog.V(2).Infof("Container %q of pod %q created with warnings: %v", container.Name, format.Pod(pod), createResp.Warnings) + } + dm.recorder.Eventf(ref, api.EventTypeNormal, kubecontainer.CreatedContainer, "Created container with docker id %v", utilstrings.ShortenString(createResp.ID, 12)) - if err = dm.client.StartContainer(dockerContainer.ID, nil); err != nil { + if err = dm.client.StartContainer(createResp.ID); err != nil { dm.recorder.Eventf(ref, api.EventTypeWarning, kubecontainer.FailedToStartContainer, - "Failed to start container with docker id %v with error: %v", utilstrings.ShortenString(dockerContainer.ID, 12), err) + "Failed to start container with docker id %v with error: %v", utilstrings.ShortenString(createResp.ID, 12), err) return kubecontainer.ContainerID{}, err } - dm.recorder.Eventf(ref, api.EventTypeNormal, kubecontainer.StartedContainer, "Started container with docker id %v", utilstrings.ShortenString(dockerContainer.ID, 12)) + dm.recorder.Eventf(ref, api.EventTypeNormal, kubecontainer.StartedContainer, "Started container with docker id %v", utilstrings.ShortenString(createResp.ID, 12)) - return kubecontainer.DockerID(dockerContainer.ID).ContainerID(), nil + return kubecontainer.DockerID(createResp.ID).ContainerID(), nil } // setInfraContainerNetworkConfig sets the network configuration for the infra-container. We only set network configuration for infra-container, all // the user containers will share the same network namespace with infra-container. -func setInfraContainerNetworkConfig(pod *api.Pod, netMode string, opts *kubecontainer.RunContainerOptions, dockerOpts docker.CreateContainerOptions) { +func setInfraContainerNetworkConfig(pod *api.Pod, netMode string, opts *kubecontainer.RunContainerOptions, dockerOpts dockertypes.ContainerCreateConfig) { exposedPorts, portBindings := makePortsAndBindings(opts.PortMappings) dockerOpts.Config.ExposedPorts = exposedPorts - dockerOpts.HostConfig.PortBindings = portBindings + dockerOpts.HostConfig.PortBindings = dockernat.PortMap(portBindings) if netMode != namespaceModeHost { dockerOpts.Config.Hostname = opts.Hostname @@ -686,16 +702,16 @@ func setInfraContainerNetworkConfig(pod *api.Pod, netMode string, opts *kubecont } } -func setEntrypointAndCommand(container *api.Container, opts *kubecontainer.RunContainerOptions, dockerOpts *docker.CreateContainerOptions) { +func setEntrypointAndCommand(container *api.Container, opts *kubecontainer.RunContainerOptions, dockerOpts dockertypes.ContainerCreateConfig) { command, args := kubecontainer.ExpandContainerCommandAndArgs(container, opts.Envs) - dockerOpts.Config.Entrypoint = command - dockerOpts.Config.Cmd = args + dockerOpts.Config.Entrypoint = dockerstrslice.StrSlice(command) + dockerOpts.Config.Cmd = dockerstrslice.StrSlice(args) } // A helper function to get the KubeletContainerName and hash from a docker // container. -func getDockerContainerNameInfo(c *docker.APIContainers) (*KubeletContainerName, uint64, error) { +func getDockerContainerNameInfo(c *dockertypes.Container) (*KubeletContainerName, uint64, error) { if len(c.Names) == 0 { return nil, 0, fmt.Errorf("cannot parse empty docker container name: %#v", c.Names) } @@ -707,7 +723,7 @@ func getDockerContainerNameInfo(c *docker.APIContainers) (*KubeletContainerName, } // Get pod UID, name, and namespace by examining the container names. -func getPodInfoFromContainer(c *docker.APIContainers) (types.UID, string, string, error) { +func getPodInfoFromContainer(c *dockertypes.Container) (types.UID, string, string, error) { dockerName, _, err := getDockerContainerNameInfo(c) if err != nil { return types.UID(""), "", "", err @@ -779,8 +795,8 @@ func (dm *DockerManager) GetPods(all bool) ([]*kubecontainer.Pod, error) { } // Convert map to list. - for _, c := range pods { - result = append(result, c) + for _, p := range pods { + result = append(result, p) } return result, nil } @@ -859,9 +875,9 @@ func readOnlyRootFilesystem(container *api.Container) bool { } // container must not be nil -func getDockerNetworkMode(container *docker.Container) string { +func getDockerNetworkMode(container *dockertypes.ContainerJSON) string { if container.HostConfig != nil { - return container.HostConfig.NetworkMode + return string(container.HostConfig.NetworkMode) } return "" } @@ -1376,7 +1392,7 @@ func (dm *DockerManager) killContainer(containerID kubecontainer.ContainerID, co if gracePeriod < minimumGracePeriodInSeconds { gracePeriod = minimumGracePeriodInSeconds } - err := dm.client.StopContainer(ID, uint(gracePeriod)) + err := dm.client.StopContainer(ID, int(gracePeriod)) if err == nil { glog.V(2).Infof("Container %q exited after %s", name, unversioned.Now().Sub(start.Time)) } else { @@ -1399,7 +1415,7 @@ func (dm *DockerManager) killContainer(containerID kubecontainer.ContainerID, co var errNoPodOnContainer = fmt.Errorf("no pod information labels on Docker container") // containerAndPodFromLabels tries to load the appropriate container info off of a Docker container's labels -func containerAndPodFromLabels(inspect *docker.Container) (pod *api.Pod, container *api.Container, err error) { +func containerAndPodFromLabels(inspect *dockertypes.ContainerJSON) (pod *api.Pod, container *api.Container, err error) { if inspect == nil && inspect.Config == nil && inspect.Config.Labels == nil { return nil, nil, errNoPodOnContainer } @@ -1438,7 +1454,7 @@ func containerAndPodFromLabels(inspect *docker.Container) (pod *api.Pod, contain return } -func (dm *DockerManager) applyOOMScoreAdj(container *api.Container, containerInfo *docker.Container) error { +func (dm *DockerManager) applyOOMScoreAdj(container *api.Container, containerInfo *dockertypes.ContainerJSON) error { cgroupName, err := dm.procFs.GetFullContainerName(containerInfo.State.Pid) if err != nil { if err == os.ErrNotExist { @@ -1544,7 +1560,7 @@ func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Containe return id, err } -func (dm *DockerManager) applyOOMScoreAdjIfNeeded(container *api.Container, containerInfo *docker.Container) error { +func (dm *DockerManager) applyOOMScoreAdjIfNeeded(container *api.Container, containerInfo *dockertypes.ContainerJSON) error { // Compare current API version with expected api version. result, err := dm.checkDockerAPIVersion(dockerv110APIVersion) if err != nil { @@ -1923,8 +1939,7 @@ func (dm *DockerManager) SyncPod(pod *api.Pod, _ api.PodStatus, podStatus *kubec } // Setup the host interface unless the pod is on the host's network (FIXME: move to networkPlugin when ready) - var podInfraContainer *docker.Container - podInfraContainer, err = dm.client.InspectContainer(string(podInfraContainerID)) + podInfraContainer, err := dm.client.InspectContainer(string(podInfraContainerID)) if err != nil { glog.Errorf("Failed to inspect pod infra container: %v; Skipping pod %q", err, format.Pod(pod)) result.Fail(err) @@ -2145,7 +2160,7 @@ func (dm *DockerManager) GetPodStatus(uid types.UID, name, namespace string) (*k // However, there may be some old containers without these labels, so at least now we can't do that. // TODO(random-liu): Do only one list and pass in the list result in the future // TODO(random-liu): Add filter when we are sure that all the containers have the labels - containers, err := dm.client.ListContainers(docker.ListContainersOptions{All: true}) + containers, err := dm.client.ListContainers(dockertypes.ContainerListOptions{All: true}) if err != nil { return podStatus, err } @@ -2166,7 +2181,7 @@ func (dm *DockerManager) GetPodStatus(uid types.UID, name, namespace string) (*k } result, ip, err := dm.inspectContainer(c.ID, name, namespace) if err != nil { - if _, ok := err.(*docker.NoSuchContainer); ok { + if _, ok := err.(containerNotFoundError); ok { // https://github.com/kubernetes/kubernetes/issues/22541 // Sometimes when docker's state is corrupt, a container can be listed // but couldn't be inspected. We fake a status for this container so diff --git a/pkg/kubelet/dockertools/manager_test.go b/pkg/kubelet/dockertools/manager_test.go index 0bf4997bbdd..c3dbcff40c3 100644 --- a/pkg/kubelet/dockertools/manager_test.go +++ b/pkg/kubelet/dockertools/manager_test.go @@ -29,6 +29,9 @@ import ( "testing" "time" + dockertypes "github.com/docker/engine-api/types" + dockercontainer "github.com/docker/engine-api/types/container" + dockerstrslice "github.com/docker/engine-api/types/strslice" docker "github.com/fsouza/go-dockerclient" cadvisorapi "github.com/google/cadvisor/info/v1" "github.com/stretchr/testify/assert" @@ -162,13 +165,13 @@ func TestSetEntrypointAndCommand(t *testing.T) { name string container *api.Container envs []kubecontainer.EnvVar - expected *docker.CreateContainerOptions + expected *dockertypes.ContainerCreateConfig }{ { name: "none", container: &api.Container{}, - expected: &docker.CreateContainerOptions{ - Config: &docker.Config{}, + expected: &dockertypes.ContainerCreateConfig{ + Config: &dockercontainer.Config{}, }, }, { @@ -176,9 +179,9 @@ func TestSetEntrypointAndCommand(t *testing.T) { container: &api.Container{ Command: []string{"foo", "bar"}, }, - expected: &docker.CreateContainerOptions{ - Config: &docker.Config{ - Entrypoint: []string{"foo", "bar"}, + expected: &dockertypes.ContainerCreateConfig{ + Config: &dockercontainer.Config{ + Entrypoint: dockerstrslice.StrSlice([]string{"foo", "bar"}), }, }, }, @@ -197,9 +200,9 @@ func TestSetEntrypointAndCommand(t *testing.T) { Value: "boo", }, }, - expected: &docker.CreateContainerOptions{ - Config: &docker.Config{ - Entrypoint: []string{"foo", "zoo", "boo"}, + expected: &dockertypes.ContainerCreateConfig{ + Config: &dockercontainer.Config{ + Entrypoint: dockerstrslice.StrSlice([]string{"foo", "zoo", "boo"}), }, }, }, @@ -208,8 +211,8 @@ func TestSetEntrypointAndCommand(t *testing.T) { container: &api.Container{ Args: []string{"foo", "bar"}, }, - expected: &docker.CreateContainerOptions{ - Config: &docker.Config{ + expected: &dockertypes.ContainerCreateConfig{ + Config: &dockercontainer.Config{ Cmd: []string{"foo", "bar"}, }, }, @@ -229,9 +232,9 @@ func TestSetEntrypointAndCommand(t *testing.T) { Value: "trap", }, }, - expected: &docker.CreateContainerOptions{ - Config: &docker.Config{ - Cmd: []string{"zap", "hap", "trap"}, + expected: &dockertypes.ContainerCreateConfig{ + Config: &dockercontainer.Config{ + Cmd: dockerstrslice.StrSlice([]string{"zap", "hap", "trap"}), }, }, }, @@ -241,10 +244,10 @@ func TestSetEntrypointAndCommand(t *testing.T) { Command: []string{"foo"}, Args: []string{"bar", "baz"}, }, - expected: &docker.CreateContainerOptions{ - Config: &docker.Config{ - Entrypoint: []string{"foo"}, - Cmd: []string{"bar", "baz"}, + expected: &dockertypes.ContainerCreateConfig{ + Config: &dockercontainer.Config{ + Entrypoint: dockerstrslice.StrSlice([]string{"foo"}), + Cmd: dockerstrslice.StrSlice([]string{"bar", "baz"}), }, }, }, @@ -268,10 +271,10 @@ func TestSetEntrypointAndCommand(t *testing.T) { Value: "roo", }, }, - expected: &docker.CreateContainerOptions{ - Config: &docker.Config{ - Entrypoint: []string{"boo--zoo", "foo", "roo"}, - Cmd: []string{"foo", "zoo", "boo"}, + expected: &dockertypes.ContainerCreateConfig{ + Config: &dockercontainer.Config{ + Entrypoint: dockerstrslice.StrSlice([]string{"boo--zoo", "foo", "roo"}), + Cmd: dockerstrslice.StrSlice([]string{"foo", "zoo", "boo"}), }, }, }, @@ -282,8 +285,8 @@ func TestSetEntrypointAndCommand(t *testing.T) { Envs: tc.envs, } - actualOpts := &docker.CreateContainerOptions{ - Config: &docker.Config{}, + actualOpts := dockertypes.ContainerCreateConfig{ + Config: &dockercontainer.Config{}, } setEntrypointAndCommand(tc.container, opts, actualOpts) @@ -319,7 +322,7 @@ func verifyPods(a, b []*kubecontainer.Pod) bool { func TestGetPods(t *testing.T) { manager, fakeDocker := newTestDockerManager() - dockerContainers := []*docker.Container{ + dockerContainers := []*FakeContainer{ { ID: "1111", Name: "/k8s_foo_qux_new_1234_42", @@ -338,7 +341,7 @@ func TestGetPods(t *testing.T) { // because the conversion is tested separately in convert_test.go containers := make([]*kubecontainer.Container, len(dockerContainers)) for i := range containers { - c, err := toRuntimeContainer(&docker.APIContainers{ + c, err := toRuntimeContainer(&dockertypes.Container{ ID: dockerContainers[i].ID, Names: []string{dockerContainers[i].Name}, }) @@ -394,40 +397,6 @@ func TestListImages(t *testing.T) { } } -func apiContainerToContainer(c docker.APIContainers) kubecontainer.Container { - dockerName, hash, err := ParseDockerName(c.Names[0]) - if err != nil { - return kubecontainer.Container{} - } - return kubecontainer.Container{ - ID: kubecontainer.ContainerID{Type: "docker", ID: c.ID}, - Name: dockerName.ContainerName, - Hash: hash, - } -} - -func dockerContainersToPod(containers []*docker.APIContainers) kubecontainer.Pod { - var pod kubecontainer.Pod - for _, c := range containers { - dockerName, hash, err := ParseDockerName(c.Names[0]) - if err != nil { - continue - } - pod.Containers = append(pod.Containers, &kubecontainer.Container{ - ID: kubecontainer.ContainerID{Type: "docker", ID: c.ID}, - Name: dockerName.ContainerName, - Hash: hash, - Image: c.Image, - }) - // TODO(yifan): Only one evaluation is enough. - pod.ID = dockerName.PodUID - name, namespace, _ := kubecontainer.ParsePodFullName(dockerName.PodFullName) - pod.Name = name - pod.Namespace = namespace - } - return pod -} - func TestKillContainerInPod(t *testing.T) { manager, fakeDocker := newTestDockerManager() @@ -439,7 +408,7 @@ func TestKillContainerInPod(t *testing.T) { }, Spec: api.PodSpec{Containers: []api.Container{{Name: "foo"}, {Name: "bar"}}}, } - containers := []*docker.Container{ + containers := []*FakeContainer{ { ID: "1111", Name: "/k8s_foo_qux_new_1234_42", @@ -498,11 +467,11 @@ func TestKillContainerInPodWithPreStop(t *testing.T) { if err != nil { t.Errorf("unexpected error: %v", err) } - containers := []*docker.Container{ + containers := []*FakeContainer{ { ID: "1111", Name: "/k8s_foo_qux_new_1234_42", - Config: &docker.Config{ + Config: &dockercontainer.Config{ Labels: map[string]string{ kubernetesPodLabel: string(podString), kubernetesContainerNameLabel: "foo", @@ -541,7 +510,7 @@ func TestKillContainerInPodWithError(t *testing.T) { }, Spec: api.PodSpec{Containers: []api.Container{{Name: "foo"}, {Name: "bar"}}}, } - containers := []*docker.Container{ + containers := []*FakeContainer{ { ID: "1111", Name: "/k8s_foo_qux_new_1234_42", @@ -634,13 +603,13 @@ func TestSyncPodCreateNetAndContainer(t *testing.T) { fakeDocker.Lock() found := false - for _, c := range fakeDocker.ContainerList { + for _, c := range fakeDocker.RunningContainerList { if c.Image == "pod_infra_image" && strings.HasPrefix(c.Names[0], "/k8s_POD") { found = true } } if !found { - t.Errorf("Custom pod infra container not found: %v", fakeDocker.ContainerList) + t.Errorf("Custom pod infra container not found: %v", fakeDocker.RunningContainerList) } if len(fakeDocker.Created) != 2 || @@ -708,7 +677,7 @@ func TestSyncPodWithPodInfraCreatesContainer(t *testing.T) { }, } - fakeDocker.SetFakeRunningContainers([]*docker.Container{{ + fakeDocker.SetFakeRunningContainers([]*FakeContainer{{ ID: "9876", // Pod infra container. Name: "/k8s_POD." + strconv.FormatUint(generatePodInfraContainerHash(pod), 16) + "_foo_new_12345678_0", @@ -742,7 +711,7 @@ func TestSyncPodDeletesWithNoPodInfraContainer(t *testing.T) { }, }, } - fakeDocker.SetFakeRunningContainers([]*docker.Container{{ + fakeDocker.SetFakeRunningContainers([]*FakeContainer{{ ID: "1234", Name: "/k8s_bar1_foo1_new_12345678_0", }}) @@ -785,7 +754,7 @@ func TestSyncPodDeletesDuplicate(t *testing.T) { }, } - fakeDocker.SetFakeRunningContainers([]*docker.Container{ + fakeDocker.SetFakeRunningContainers([]*FakeContainer{ { ID: "1234", Name: "/k8s_foo_bar_new_12345678_1111", @@ -826,7 +795,7 @@ func TestSyncPodBadHash(t *testing.T) { }, } - fakeDocker.SetFakeRunningContainers([]*docker.Container{ + fakeDocker.SetFakeRunningContainers([]*FakeContainer{ { ID: "1234", Name: "/k8s_bar.1234_foo_new_12345678_42", @@ -864,7 +833,7 @@ func TestSyncPodsUnhealthy(t *testing.T) { }, } - fakeDocker.SetFakeRunningContainers([]*docker.Container{ + fakeDocker.SetFakeRunningContainers([]*FakeContainer{ { ID: unhealthyContainerID, Name: "/k8s_unhealthy_foo_new_12345678_42", @@ -904,7 +873,7 @@ func TestSyncPodsDoesNothing(t *testing.T) { }, }, } - fakeDocker.SetFakeRunningContainers([]*docker.Container{ + fakeDocker.SetFakeRunningContainers([]*FakeContainer{ { ID: "1234", Name: "/k8s_bar." + strconv.FormatUint(kubecontainer.HashContainer(&container), 16) + "_foo_new_12345678_0", @@ -935,35 +904,26 @@ func TestSyncPodWithRestartPolicy(t *testing.T) { Containers: containers, }, } - dockerContainers := []*docker.Container{ + dockerContainers := []*FakeContainer{ { - ID: "9876", - Name: "/k8s_POD." + strconv.FormatUint(generatePodInfraContainerHash(pod), 16) + "_foo_new_12345678_0", - Config: &docker.Config{}, - State: docker.State{ - StartedAt: time.Now(), - Running: true, - }, + ID: "9876", + Name: "/k8s_POD." + strconv.FormatUint(generatePodInfraContainerHash(pod), 16) + "_foo_new_12345678_0", + StartedAt: time.Now(), + Running: true, }, { - ID: "1234", - Name: "/k8s_succeeded." + strconv.FormatUint(kubecontainer.HashContainer(&containers[0]), 16) + "_foo_new_12345678_0", - Config: &docker.Config{}, - State: docker.State{ - ExitCode: 0, - StartedAt: time.Now(), - FinishedAt: time.Now(), - }, + ID: "1234", + Name: "/k8s_succeeded." + strconv.FormatUint(kubecontainer.HashContainer(&containers[0]), 16) + "_foo_new_12345678_0", + ExitCode: 0, + StartedAt: time.Now(), + FinishedAt: time.Now(), }, { - ID: "5678", - Name: "/k8s_failed." + strconv.FormatUint(kubecontainer.HashContainer(&containers[1]), 16) + "_foo_new_12345678_0", - Config: &docker.Config{}, - State: docker.State{ - ExitCode: 42, - StartedAt: time.Now(), - FinishedAt: time.Now(), - }, + ID: "5678", + Name: "/k8s_failed." + strconv.FormatUint(kubecontainer.HashContainer(&containers[1]), 16) + "_foo_new_12345678_0", + ExitCode: 42, + StartedAt: time.Now(), + FinishedAt: time.Now(), }} tests := []struct { @@ -1040,31 +1000,25 @@ func TestSyncPodBackoff(t *testing.T) { } stableId := "k8s_bad." + strconv.FormatUint(kubecontainer.HashContainer(&containers[1]), 16) + "_podfoo_nsnew_12345678" - dockerContainers := []*docker.Container{ + dockerContainers := []*FakeContainer{ { - ID: "9876", - Name: "/k8s_POD." + strconv.FormatUint(generatePodInfraContainerHash(pod), 16) + "_podfoo_nsnew_12345678_0", - State: docker.State{ - StartedAt: startTime, - Running: true, - }, + ID: "9876", + Name: "/k8s_POD." + strconv.FormatUint(generatePodInfraContainerHash(pod), 16) + "_podfoo_nsnew_12345678_0", + StartedAt: startTime, + Running: true, }, { - ID: "1234", - Name: "/k8s_good." + strconv.FormatUint(kubecontainer.HashContainer(&containers[0]), 16) + "_podfoo_nsnew_12345678_0", - State: docker.State{ - StartedAt: startTime, - Running: true, - }, + ID: "1234", + Name: "/k8s_good." + strconv.FormatUint(kubecontainer.HashContainer(&containers[0]), 16) + "_podfoo_nsnew_12345678_0", + StartedAt: startTime, + Running: true, }, { - ID: "5678", - Name: "/k8s_bad." + strconv.FormatUint(kubecontainer.HashContainer(&containers[1]), 16) + "_podfoo_nsnew_12345678_0", - State: docker.State{ - ExitCode: 42, - StartedAt: startTime, - FinishedAt: fakeClock.Now(), - }, + ID: "5678", + Name: "/k8s_bad." + strconv.FormatUint(kubecontainer.HashContainer(&containers[1]), 16) + "_podfoo_nsnew_12345678_0", + ExitCode: 42, + StartedAt: startTime, + FinishedAt: fakeClock.Now(), }, } @@ -1113,7 +1067,7 @@ func TestSyncPodBackoff(t *testing.T) { if len(fakeDocker.Created) > 0 { // pretend kill the container fakeDocker.Created = nil - dockerContainers[2].State.FinishedAt = startTime.Add(time.Duration(c.killDelay) * time.Second) + dockerContainers[2].FinishedAt = startTime.Add(time.Duration(c.killDelay) * time.Second) } } } @@ -1199,7 +1153,7 @@ func TestGetRestartCount(t *testing.T) { verifyRestartCount(&pod, 3) // All exited containers have been garbage collected, restart count should be got from old api pod status - fakeDocker.ExitedContainerList = []docker.APIContainers{} + fakeDocker.ExitedContainerList = []dockertypes.Container{} verifyRestartCount(&pod, 3) killOneContainer(&pod) @@ -1228,12 +1182,12 @@ func TestGetTerminationMessagePath(t *testing.T) { runSyncPod(t, dm, fakeDocker, pod, nil, false) - containerList := fakeDocker.ContainerList + containerList := fakeDocker.RunningContainerList if len(containerList) != 2 { // One for infra container, one for container "bar" t.Fatalf("unexpected container list length %d", len(containerList)) } - inspectResult, err := dm.client.InspectContainer(containerList[0].ID) + inspectResult, err := fakeDocker.InspectContainer(containerList[0].ID) if err != nil { t.Fatalf("unexpected inspect error: %v", err) } @@ -1271,7 +1225,7 @@ func TestSyncPodWithPodInfraCreatesContainerCallsHandler(t *testing.T) { }, }, } - fakeDocker.SetFakeRunningContainers([]*docker.Container{{ + fakeDocker.SetFakeRunningContainers([]*FakeContainer{{ ID: "9876", Name: "/k8s_POD." + strconv.FormatUint(generatePodInfraContainerHash(pod), 16) + "_foo_new_12345678_0", }}) @@ -1321,7 +1275,7 @@ func TestSyncPodEventHandlerFails(t *testing.T) { }, } - fakeDocker.SetFakeRunningContainers([]*docker.Container{{ + fakeDocker.SetFakeRunningContainers([]*FakeContainer{{ ID: "9876", Name: "/k8s_POD." + strconv.FormatUint(generatePodInfraContainerHash(pod), 16) + "_foo_new_12345678_0", }}) @@ -1727,7 +1681,7 @@ func TestSyncPodWithFailure(t *testing.T) { puller.HasImages = []string{test.container.Image} // Pretend that the pod infra container has already been created, so that // we can run the user containers. - fakeDocker.SetFakeRunningContainers([]*docker.Container{{ + fakeDocker.SetFakeRunningContainers([]*FakeContainer{{ ID: "9876", Name: "/k8s_POD." + strconv.FormatUint(generatePodInfraContainerHash(pod), 16) + "_foo_new_12345678_0", }}) @@ -1911,29 +1865,25 @@ func TestGetPodStatusNoSuchContainer(t *testing.T) { }, } - fakeDocker.SetFakeContainers([]*docker.Container{ + fakeDocker.SetFakeContainers([]*FakeContainer{ { - ID: noSuchContainerID, - Name: "/k8s_nosuchcontainer_foo_new_12345678_42", - State: docker.State{ - ExitCode: 0, - StartedAt: time.Now(), - FinishedAt: time.Now(), - Running: false, - }, + ID: noSuchContainerID, + Name: "/k8s_nosuchcontainer_foo_new_12345678_42", + ExitCode: 0, + StartedAt: time.Now(), + FinishedAt: time.Now(), + Running: false, }, { - ID: infraContainerID, - Name: "/k8s_POD." + strconv.FormatUint(generatePodInfraContainerHash(pod), 16) + "_foo_new_12345678_42", - State: docker.State{ - ExitCode: 0, - StartedAt: time.Now(), - FinishedAt: time.Now(), - Running: false, - }, - }}) - - fakeDocker.InjectErrors(map[string]error{"inspect": &docker.NoSuchContainer{}}) + ID: infraContainerID, + Name: "/k8s_POD." + strconv.FormatUint(generatePodInfraContainerHash(pod), 16) + "_foo_new_12345678_42", + ExitCode: 0, + StartedAt: time.Now(), + FinishedAt: time.Now(), + Running: false, + }, + }) + fakeDocker.InjectErrors(map[string]error{"inspect_container": containerNotFoundError{}}) runSyncPod(t, dm, fakeDocker, pod, nil, false) // Verify that we will try to start new contrainers even if the inspections diff --git a/pkg/kubelet/network/cni/cni_test.go b/pkg/kubelet/network/cni/cni_test.go index 3d2179a0d96..dbf237d46ea 100644 --- a/pkg/kubelet/network/cni/cni_test.go +++ b/pkg/kubelet/network/cni/cni_test.go @@ -30,7 +30,6 @@ import ( clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - docker "github.com/fsouza/go-dockerclient" cadvisorapi "github.com/google/cadvisor/info/v1" "k8s.io/kubernetes/cmd/kubelet/app/options" @@ -132,10 +131,10 @@ func (fnh *fakeNetworkHost) GetKubeClient() clientset.Interface { func (nh *fakeNetworkHost) GetRuntime() kubecontainer.Runtime { dm, fakeDockerClient := newTestDockerManager() - fakeDockerClient.SetFakeRunningContainers([]*docker.Container{ + fakeDockerClient.SetFakeRunningContainers([]*dockertools.FakeContainer{ { - ID: "test_infra_container", - State: docker.State{Pid: 12345}, + ID: "test_infra_container", + Pid: 12345, }, }) return dm diff --git a/pkg/securitycontext/fake.go b/pkg/securitycontext/fake.go index 5c77f525c64..804f1690599 100644 --- a/pkg/securitycontext/fake.go +++ b/pkg/securitycontext/fake.go @@ -19,7 +19,7 @@ package securitycontext import ( "k8s.io/kubernetes/pkg/api" - docker "github.com/fsouza/go-dockerclient" + dockercontainer "github.com/docker/engine-api/types/container" ) // ValidSecurityContextWithContainerDefaults creates a valid security context provider based on @@ -39,7 +39,7 @@ func NewFakeSecurityContextProvider() SecurityContextProvider { type FakeSecurityContextProvider struct{} -func (p FakeSecurityContextProvider) ModifyContainerConfig(pod *api.Pod, container *api.Container, config *docker.Config) { +func (p FakeSecurityContextProvider) ModifyContainerConfig(pod *api.Pod, container *api.Container, config *dockercontainer.Config) { } -func (p FakeSecurityContextProvider) ModifyHostConfig(pod *api.Pod, container *api.Container, hostConfig *docker.HostConfig) { +func (p FakeSecurityContextProvider) ModifyHostConfig(pod *api.Pod, container *api.Container, hostConfig *dockercontainer.HostConfig) { } diff --git a/pkg/securitycontext/provider.go b/pkg/securitycontext/provider.go index 9bd5b16b62c..f37aa4e0179 100644 --- a/pkg/securitycontext/provider.go +++ b/pkg/securitycontext/provider.go @@ -23,7 +23,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/kubelet/leaky" - docker "github.com/fsouza/go-dockerclient" + dockercontainer "github.com/docker/engine-api/types/container" ) // NewSimpleSecurityContextProvider creates a new SimpleSecurityContextProvider. @@ -37,7 +37,7 @@ type SimpleSecurityContextProvider struct{} // ModifyContainerConfig is called before the Docker createContainer call. // The security context provider can make changes to the Config with which // the container is created. -func (p SimpleSecurityContextProvider) ModifyContainerConfig(pod *api.Pod, container *api.Container, config *docker.Config) { +func (p SimpleSecurityContextProvider) ModifyContainerConfig(pod *api.Pod, container *api.Container, config *dockercontainer.Config) { effectiveSC := DetermineEffectiveSecurityContext(pod, container) if effectiveSC == nil { return @@ -50,7 +50,7 @@ func (p SimpleSecurityContextProvider) ModifyContainerConfig(pod *api.Pod, conta // ModifyHostConfig is called before the Docker runContainer call. // The security context provider can make changes to the HostConfig, affecting // security options, whether the container is privileged, volume binds, etc. -func (p SimpleSecurityContextProvider) ModifyHostConfig(pod *api.Pod, container *api.Container, hostConfig *docker.HostConfig) { +func (p SimpleSecurityContextProvider) ModifyHostConfig(pod *api.Pod, container *api.Container, hostConfig *dockercontainer.HostConfig) { // Apply pod security context if container.Name != leaky.PodInfraContainerName && pod.Spec.SecurityContext != nil { // TODO: We skip application of supplemental groups to the diff --git a/pkg/securitycontext/provider_test.go b/pkg/securitycontext/provider_test.go index 062f0e7f7c9..5d70c338de0 100644 --- a/pkg/securitycontext/provider_test.go +++ b/pkg/securitycontext/provider_test.go @@ -22,7 +22,7 @@ import ( "strconv" "testing" - docker "github.com/fsouza/go-dockerclient" + dockercontainer "github.com/docker/engine-api/types/container" "k8s.io/kubernetes/pkg/api" apitesting "k8s.io/kubernetes/pkg/api/testing" ) @@ -35,28 +35,28 @@ func TestModifyContainerConfig(t *testing.T) { name string podSc *api.PodSecurityContext sc *api.SecurityContext - expected *docker.Config + expected *dockercontainer.Config }{ { name: "container.SecurityContext.RunAsUser set", sc: &api.SecurityContext{ RunAsUser: &uid, }, - expected: &docker.Config{ + expected: &dockercontainer.Config{ User: strconv.FormatInt(uid, 10), }, }, { name: "no RunAsUser value set", sc: &api.SecurityContext{}, - expected: &docker.Config{}, + expected: &dockercontainer.Config{}, }, { name: "pod.Spec.SecurityContext.RunAsUser set", podSc: &api.PodSecurityContext{ RunAsUser: &uid, }, - expected: &docker.Config{ + expected: &dockercontainer.Config{ User: strconv.FormatInt(uid, 10), }, }, @@ -68,7 +68,7 @@ func TestModifyContainerConfig(t *testing.T) { sc: &api.SecurityContext{ RunAsUser: &overrideUid, }, - expected: &docker.Config{ + expected: &dockercontainer.Config{ User: strconv.FormatInt(overrideUid, 10), }, }, @@ -79,7 +79,7 @@ func TestModifyContainerConfig(t *testing.T) { for _, tc := range cases { pod := &api.Pod{Spec: api.PodSpec{SecurityContext: tc.podSc}} dummyContainer.SecurityContext = tc.sc - dockerCfg := &docker.Config{} + dockerCfg := &dockercontainer.Config{} provider.ModifyContainerConfig(pod, dummyContainer, dockerCfg) @@ -93,16 +93,16 @@ func TestModifyHostConfig(t *testing.T) { priv := true setPrivSC := &api.SecurityContext{} setPrivSC.Privileged = &priv - setPrivHC := &docker.HostConfig{ + setPrivHC := &dockercontainer.HostConfig{ Privileged: true, } - setCapsHC := &docker.HostConfig{ + setCapsHC := &dockercontainer.HostConfig{ CapAdd: []string{"addCapA", "addCapB"}, CapDrop: []string{"dropCapA", "dropCapB"}, } - setSELinuxHC := &docker.HostConfig{} + setSELinuxHC := &dockercontainer.HostConfig{} setSELinuxHC.SecurityOpt = []string{ fmt.Sprintf("%s:%s", dockerLabelUser, "user"), fmt.Sprintf("%s:%s", dockerLabelRole, "role"), @@ -117,7 +117,7 @@ func TestModifyHostConfig(t *testing.T) { name string podSc *api.PodSecurityContext sc *api.SecurityContext - expected *docker.HostConfig + expected *dockercontainer.HostConfig }{ { name: "fully set container.SecurityContext", @@ -164,7 +164,7 @@ func TestModifyHostConfig(t *testing.T) { for _, tc := range cases { pod := &api.Pod{Spec: api.PodSpec{SecurityContext: tc.podSc}} dummyContainer.SecurityContext = tc.sc - dockerCfg := &docker.HostConfig{} + dockerCfg := &dockercontainer.HostConfig{} provider.ModifyHostConfig(pod, dummyContainer, dockerCfg) @@ -187,7 +187,7 @@ func TestModifyHostConfigPodSecurityContext(t *testing.T) { testCases := map[string]struct { securityContext *api.PodSecurityContext - expected *docker.HostConfig + expected *dockercontainer.HostConfig }{ "nil": { securityContext: nil, @@ -219,7 +219,7 @@ func TestModifyHostConfigPodSecurityContext(t *testing.T) { for k, v := range testCases { dummyPod.Spec.SecurityContext = v.securityContext - dockerCfg := &docker.HostConfig{} + dockerCfg := &dockercontainer.HostConfig{} provider.ModifyHostConfig(dummyPod, dummyContainer, dockerCfg) if !reflect.DeepEqual(v.expected, dockerCfg) { t.Errorf("unexpected modification of host config for %s. Expected: %#v Got: %#v", k, v.expected, dockerCfg) @@ -301,8 +301,8 @@ func inputSELinuxOptions() *api.SELinuxOptions { } } -func fullValidHostConfig() *docker.HostConfig { - return &docker.HostConfig{ +func fullValidHostConfig() *dockercontainer.HostConfig { + return &dockercontainer.HostConfig{ Privileged: true, CapAdd: []string{"addCapA", "addCapB"}, CapDrop: []string{"dropCapA", "dropCapB"}, diff --git a/pkg/securitycontext/types.go b/pkg/securitycontext/types.go index 61549cc00eb..6f45b19684f 100644 --- a/pkg/securitycontext/types.go +++ b/pkg/securitycontext/types.go @@ -19,21 +19,21 @@ package securitycontext import ( "k8s.io/kubernetes/pkg/api" - docker "github.com/fsouza/go-dockerclient" + dockercontainer "github.com/docker/engine-api/types/container" ) type SecurityContextProvider interface { // ModifyContainerConfig is called before the Docker createContainer call. // The security context provider can make changes to the Config with which // the container is created. - ModifyContainerConfig(pod *api.Pod, container *api.Container, config *docker.Config) + ModifyContainerConfig(pod *api.Pod, container *api.Container, config *dockercontainer.Config) // ModifyHostConfig is called before the Docker createContainer call. // The security context provider can make changes to the HostConfig, affecting // security options, whether the container is privileged, volume binds, etc. // An error is returned if it's not possible to secure the container as requested // with a security context. - ModifyHostConfig(pod *api.Pod, container *api.Container, hostConfig *docker.HostConfig) + ModifyHostConfig(pod *api.Pod, container *api.Container, hostConfig *dockercontainer.HostConfig) } const (