Merge pull request #15079 from gmarek/fix-kubemark

Turn on smarter FakeDockerClient in HollowNode.

cmd/kubemark/hollow-node.go

@@ -107,6 +107,8 @@ func main() {
 	configFilePath := makeTempDirOrDie("config", testRootDir)
 	glog.Infof("Using %s as root dir for hollow-kubelet", testRootDir)
 	fakeDockerClient.VersionInfo = docker.Env{"ApiVersion=1.18"}
+	fakeDockerClient.ContainerMap = make(map[string]*docker.Container)
+	fakeDockerClient.EnableSleep = true
 	kcfg := kubeletapp.SimpleKubelet(
 		cl,
 		&fakeDockerClient,

pkg/kubelet/dockertools/fake_docker_client.go

@@ -19,6 +19,7 @@ package dockertools
 import (
 	"encoding/json"
 	"fmt"
+	"math/rand"
 	"os"
 	"reflect"
 	"sort"
@@ -52,6 +53,7 @@ type FakeDockerClient struct {
 	Information         docker.Env
 	ExecInspect         *docker.ExecInspect
 	execCmd             []string
+	EnableSleep         bool
 }
 
 func (f *FakeDockerClient) ClearCalls() {
@@ -175,27 +177,41 @@ func (f *FakeDockerClient) InspectImage(name string) (*docker.Image, error) {
 	return f.Image, err
 }
 
+// Sleeps random amount of time with the normal distribution with given mean and stddev
+// (in milliseconds), we never sleep less than cutOffMillis
+func (f *FakeDockerClient) normalSleep(mean, stdDev, cutOffMillis int) {
+	if !f.EnableSleep {
+		return
+	}
+	cutoff := (time.Duration)(cutOffMillis) * time.Millisecond
+	delay := (time.Duration)(rand.NormFloat64()*float64(stdDev)+float64(mean)) * time.Millisecond
+	if delay < cutoff {
+		delay = cutoff
+	}
+	time.Sleep(delay)
+}
+
 // CreateContainer is a test-spy implementation of DockerInterface.CreateContainer.
 // It adds an entry "create" to the internal method call record.
 func (f *FakeDockerClient) CreateContainer(c docker.CreateContainerOptions) (*docker.Container, error) {
 	f.Lock()
 	defer f.Unlock()
 	f.called = append(f.called, "create")
-	err := f.popError("create")
+	if err := f.popError("create"); err != nil {
-	if err == nil {
+		return nil, err
-		f.Created = append(f.Created, c.Name)
-		// This is not a very good fake. We'll just add this container's name to the list.
-		// Docker likes to add a '/', so copy that behavior.
-		name := "/" + c.Name
-		f.ContainerList = append(f.ContainerList, docker.APIContainers{ID: name, Names: []string{name}, Image: c.Config.Image})
-		container := docker.Container{ID: name, Name: name, Config: c.Config}
-		if f.ContainerMap != nil {
-			containerCopy := container
-			f.ContainerMap[name] = &containerCopy
-		}
-		return &container, nil
 	}
-	return nil, err
+	f.Created = append(f.Created, c.Name)
+	// This is not a very good fake. We'll just add this container's name to the list.
+	// Docker likes to add a '/', so copy that behavior.
+	name := "/" + c.Name
+	f.ContainerList = append(f.ContainerList, docker.APIContainers{ID: name, Names: []string{name}, Image: c.Config.Image})
+	container := docker.Container{ID: name, Name: name, Config: c.Config}
+	if f.ContainerMap != nil {
+		containerCopy := container
+		f.ContainerMap[name] = &containerCopy
+	}
+	f.normalSleep(200, 50, 50)
+	return &container, nil
 }
 
 // StartContainer is a test-spy implementation of DockerInterface.StartContainer.
@@ -204,37 +220,37 @@ func (f *FakeDockerClient) StartContainer(id string, hostConfig *docker.HostConf
 	f.Lock()
 	defer f.Unlock()
 	f.called = append(f.called, "start")
-	err := f.popError("start")
+	if err := f.popError("start"); err != nil {
-	if err == nil {
+		return err
-
-		f.Container = &docker.Container{
-			ID:         id,
-			Name:       id, // For testing purpose, we set name to id
-			Config:     &docker.Config{Image: "testimage"},
-			HostConfig: hostConfig,
-			State: docker.State{
-				Running:   true,
-				Pid:       os.Getpid(),
-				StartedAt: time.Now(),
-			},
-			NetworkSettings: &docker.NetworkSettings{IPAddress: "1.2.3.4"},
-		}
-		if f.ContainerMap != nil {
-			container, ok := f.ContainerMap[id]
-			if !ok {
-				container = &docker.Container{ID: id, Name: id}
-			}
-			container.HostConfig = hostConfig
-			container.State = docker.State{
-				Running:   true,
-				Pid:       os.Getpid(),
-				StartedAt: time.Now(),
-			}
-			container.NetworkSettings = &docker.NetworkSettings{IPAddress: "2.3.4.5"}
-			f.ContainerMap[id] = container
-		}
 	}
-	return err
+	f.Container = &docker.Container{
+		ID:         id,
+		Name:       id, // For testing purpose, we set name to id
+		Config:     &docker.Config{Image: "testimage"},
+		HostConfig: hostConfig,
+		State: docker.State{
+			Running:   true,
+			Pid:       os.Getpid(),
+			StartedAt: time.Now(),
+		},
+		NetworkSettings: &docker.NetworkSettings{IPAddress: "1.2.3.4"},
+	}
+	if f.ContainerMap != nil {
+		container, ok := f.ContainerMap[id]
+		if !ok {
+			container = &docker.Container{ID: id, Name: id}
+		}
+		container.HostConfig = hostConfig
+		container.State = docker.State{
+			Running:   true,
+			Pid:       os.Getpid(),
+			StartedAt: time.Now(),
+		}
+		container.NetworkSettings = &docker.NetworkSettings{IPAddress: "2.3.4.5"}
+		f.ContainerMap[id] = container
+	}
+	f.normalSleep(200, 50, 50)
+	return nil
 }
 
 // StopContainer is a test-spy implementation of DockerInterface.StopContainer.
@@ -243,39 +259,39 @@ func (f *FakeDockerClient) StopContainer(id string, timeout uint) error {
 	f.Lock()
 	defer f.Unlock()
 	f.called = append(f.called, "stop")
-	err := f.popError("stop")
+	if err := f.popError("stop"); err != nil {
-	if err == nil {
+		return err
-		f.Stopped = append(f.Stopped, id)
-		var newList []docker.APIContainers
-		for _, container := range f.ContainerList {
-			if container.ID == id {
-				f.ExitedContainerList = append(f.ExitedContainerList, container)
-				continue
-			}
-			newList = append(newList, container)
-		}
-		f.ContainerList = newList
-		if f.ContainerMap != nil {
-			container, ok := f.ContainerMap[id]
-			if !ok {
-				container = &docker.Container{
-					ID:   id,
-					Name: id,
-					State: docker.State{
-						Running:    false,
-						StartedAt:  time.Now().Add(-time.Second),
-						FinishedAt: time.Now(),
-					},
-				}
-			} else {
-				container.State.FinishedAt = time.Now()
-				container.State.Running = false
-			}
-			f.ContainerMap[id] = container
-		}
-
 	}
-	return err
+	f.Stopped = append(f.Stopped, id)
+	var newList []docker.APIContainers
+	for _, container := range f.ContainerList {
+		if container.ID == id {
+			f.ExitedContainerList = append(f.ExitedContainerList, container)
+			continue
+		}
+		newList = append(newList, container)
+	}
+	f.ContainerList = newList
+	if f.ContainerMap != nil {
+		container, ok := f.ContainerMap[id]
+		if !ok {
+			container = &docker.Container{
+				ID:   id,
+				Name: id,
+				State: docker.State{
+					Running:    false,
+					StartedAt:  time.Now().Add(-time.Second),
+					FinishedAt: time.Now(),
+				},
+			}
+		} else {
+			container.State.FinishedAt = time.Now()
+			container.State.Running = false
+		}
+		f.ContainerMap[id] = container
+	}
+	f.normalSleep(200, 50, 50)
+	return nil
 }
 
 func (f *FakeDockerClient) RemoveContainer(opts docker.RemoveContainerOptions) error {