diff --git a/pkg/kubelet/dockertools/docker_test.go b/pkg/kubelet/dockertools/docker_test.go index 1a6143bc9e9..30eae08a2e7 100644 --- a/pkg/kubelet/dockertools/docker_test.go +++ b/pkg/kubelet/dockertools/docker_test.go @@ -243,7 +243,7 @@ func TestPull(t *testing.T) { func TestDockerKeyringLookupFails(t *testing.T) { fakeKeyring := &credentialprovider.FakeKeyring{} fakeClient := &FakeDockerClient{ - Err: fmt.Errorf("test error"), + Errors: map[string]error{"pull": fmt.Errorf("test error")}, } dp := dockerPuller{ @@ -394,7 +394,7 @@ func TestIsImagePresent(t *testing.T) { } func TestGetRunningContainers(t *testing.T) { - fakeDocker := &FakeDockerClient{} + fakeDocker := &FakeDockerClient{Errors: make(map[string]error)} fakeRecorder := &record.FakeRecorder{} containerManager := NewDockerManager(fakeDocker, fakeRecorder, PodInfraContainerImage) tests := []struct { @@ -478,14 +478,16 @@ func TestGetRunningContainers(t *testing.T) { } for _, test := range tests { fakeDocker.ContainerMap = test.containers - fakeDocker.Err = test.err + if test.err != nil { + fakeDocker.Errors["inspect_container"] = test.err + } if results, err := containerManager.GetRunningContainers(test.inputIDs); err == nil { resultIDs := []string{} for _, result := range results { resultIDs = append(resultIDs, result.ID) } if !reflect.DeepEqual(resultIDs, test.expectedIDs) { - t.Errorf("expected: %v, saw: %v", test.expectedIDs, resultIDs) + t.Errorf("expected: %#v, saw: %#v", test.expectedIDs, resultIDs) } if err != nil { t.Errorf("unexpected error: %v", err) diff --git a/pkg/kubelet/dockertools/fake_docker_client.go b/pkg/kubelet/dockertools/fake_docker_client.go index 3298baea279..f94ba324d5a 100644 --- a/pkg/kubelet/dockertools/fake_docker_client.go +++ b/pkg/kubelet/dockertools/fake_docker_client.go @@ -38,7 +38,7 @@ type FakeDockerClient struct { ContainerMap map[string]*docker.Container Image *docker.Image Images []docker.APIImages - Err error + Errors map[string]error called []string Stopped []string pulled []string @@ -118,17 +118,30 @@ func (f *FakeDockerClient) AssertUnorderedCalls(calls []string) (err error) { return } +func (f *FakeDockerClient) popError(op string) error { + if f.Errors == nil { + return nil + } + err, ok := f.Errors[op] + if ok { + delete(f.Errors, op) + return err + } else { + return nil + } +} + // ListContainers is a test-spy implementation of DockerInterface.ListContainers. // It adds an entry "list" to the internal method call record. func (f *FakeDockerClient) ListContainers(options docker.ListContainersOptions) ([]docker.APIContainers, error) { f.Lock() defer f.Unlock() f.called = append(f.called, "list") - + err := f.popError("list") if options.All { - return append(f.ContainerList, f.ExitedContainerList...), f.Err + return append(f.ContainerList, f.ExitedContainerList...), err } - return f.ContainerList, f.Err + return f.ContainerList, err } // InspectContainer is a test-spy implementation of DockerInterface.InspectContainer. @@ -137,12 +150,13 @@ func (f *FakeDockerClient) InspectContainer(id string) (*docker.Container, error f.Lock() defer f.Unlock() f.called = append(f.called, "inspect_container") + err := f.popError("inspect_container") if f.ContainerMap != nil { if container, ok := f.ContainerMap[id]; ok { - return container, f.Err + return container, err } } - return f.Container, f.Err + return f.Container, err } // InspectImage is a test-spy implementation of DockerInterface.InspectImage. @@ -151,7 +165,8 @@ func (f *FakeDockerClient) InspectImage(name string) (*docker.Image, error) { f.Lock() defer f.Unlock() f.called = append(f.called, "inspect_image") - return f.Image, f.Err + err := f.popError("inspect_image") + return f.Image, err } // CreateContainer is a test-spy implementation of DockerInterface.CreateContainer. @@ -160,12 +175,16 @@ func (f *FakeDockerClient) CreateContainer(c docker.CreateContainerOptions) (*do f.Lock() defer f.Unlock() f.called = append(f.called, "create") - f.Created = append(f.Created, c.Name) - // This is not a very good fake. We'll just add this container's name to the list. - // Docker likes to add a '/', so copy that behavior. - name := "/" + c.Name - f.ContainerList = append(f.ContainerList, docker.APIContainers{ID: name, Names: []string{name}, Image: c.Config.Image}) - return &docker.Container{ID: name}, nil + err := f.popError("create") + if err == nil { + f.Created = append(f.Created, c.Name) + // This is not a very good fake. We'll just add this container's name to the list. + // Docker likes to add a '/', so copy that behavior. + name := "/" + c.Name + f.ContainerList = append(f.ContainerList, docker.APIContainers{ID: name, Names: []string{name}, Image: c.Config.Image}) + return &docker.Container{ID: name}, nil + } + return nil, err } // StartContainer is a test-spy implementation of DockerInterface.StartContainer. @@ -174,18 +193,22 @@ func (f *FakeDockerClient) StartContainer(id string, hostConfig *docker.HostConf f.Lock() defer f.Unlock() f.called = append(f.called, "start") - f.Container = &docker.Container{ - ID: id, - Name: id, // For testing purpose, we set name to id - Config: &docker.Config{Image: "testimage"}, - HostConfig: hostConfig, - State: docker.State{ - Running: true, - Pid: os.Getpid(), - }, - NetworkSettings: &docker.NetworkSettings{IPAddress: "1.2.3.4"}, + err := f.popError("start") + if err == nil { + + f.Container = &docker.Container{ + ID: id, + Name: id, // For testing purpose, we set name to id + Config: &docker.Config{Image: "testimage"}, + HostConfig: hostConfig, + State: docker.State{ + Running: true, + Pid: os.Getpid(), + }, + NetworkSettings: &docker.NetworkSettings{IPAddress: "1.2.3.4"}, + } } - return f.Err + return err } // StopContainer is a test-spy implementation of DockerInterface.StopContainer. @@ -194,23 +217,29 @@ func (f *FakeDockerClient) StopContainer(id string, timeout uint) error { f.Lock() defer f.Unlock() f.called = append(f.called, "stop") - f.Stopped = append(f.Stopped, id) - var newList []docker.APIContainers - for _, container := range f.ContainerList { - if container.ID != id { - newList = append(newList, container) + err := f.popError("stop") + if err == nil { + f.Stopped = append(f.Stopped, id) + var newList []docker.APIContainers + for _, container := range f.ContainerList { + if container.ID != id { + newList = append(newList, container) + } } + f.ContainerList = newList } - f.ContainerList = newList - return f.Err + return err } func (f *FakeDockerClient) RemoveContainer(opts docker.RemoveContainerOptions) error { f.Lock() defer f.Unlock() f.called = append(f.called, "remove") - f.Removed = append(f.Removed, opts.ID) - return f.Err + err := f.popError("remove") + if err == nil { + f.Removed = append(f.Removed, opts.ID) + } + return err } // Logs is a test-spy implementation of DockerInterface.Logs. @@ -219,7 +248,7 @@ func (f *FakeDockerClient) Logs(opts docker.LogsOptions) error { f.Lock() defer f.Unlock() f.called = append(f.called, "logs") - return f.Err + return f.popError("logs") } // PullImage is a test-spy implementation of DockerInterface.StopContainer. @@ -228,12 +257,15 @@ func (f *FakeDockerClient) PullImage(opts docker.PullImageOptions, auth docker.A f.Lock() defer f.Unlock() f.called = append(f.called, "pull") - registry := opts.Registry - if len(registry) != 0 { - registry = registry + "/" + err := f.popError("pull") + if err == nil { + registry := opts.Registry + if len(registry) != 0 { + registry = registry + "/" + } + f.pulled = append(f.pulled, fmt.Sprintf("%s%s:%s", registry, opts.Repository, opts.Tag)) } - f.pulled = append(f.pulled, fmt.Sprintf("%s%s:%s", registry, opts.Repository, opts.Tag)) - return f.Err + return err } func (f *FakeDockerClient) Version() (*docker.Env, error) { @@ -249,12 +281,16 @@ func (f *FakeDockerClient) StartExec(_ string, _ docker.StartExecOptions) error } func (f *FakeDockerClient) ListImages(opts docker.ListImagesOptions) ([]docker.APIImages, error) { - return f.Images, f.Err + err := f.popError("list_images") + return f.Images, err } func (f *FakeDockerClient) RemoveImage(image string) error { - f.RemovedImages.Insert(image) - return f.Err + err := f.popError("remove_image") + if err == nil { + f.RemovedImages.Insert(image) + } + return err } // FakeDockerPuller is a stub implementation of DockerPuller. diff --git a/pkg/kubelet/dockertools/manager.go b/pkg/kubelet/dockertools/manager.go index a0c889df323..25a0c9e5f67 100644 --- a/pkg/kubelet/dockertools/manager.go +++ b/pkg/kubelet/dockertools/manager.go @@ -25,14 +25,21 @@ import ( "path" "strconv" "strings" + "sync" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities" "github.com/GoogleCloudPlatform/kubernetes/pkg/client/record" kubecontainer "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/container" + "github.com/GoogleCloudPlatform/kubernetes/pkg/types" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/fsouza/go-dockerclient" "github.com/golang/glog" + "github.com/golang/groupcache/lru" +) + +const ( + maxReasonCacheEntries = 200 ) // Implements kubecontainer.ContainerRunner. @@ -44,13 +51,60 @@ type DockerManager struct { // TODO(yifan): PodInfraContainerImage can be unexported once // we move createPodInfraContainer into dockertools. PodInfraContainerImage string + // reasonCache stores the failure reason of the last container creation + // and/or start in a string, keyed by _. The goal + // is to propagate this reason to the container status. This endeavor is + // "best-effort" for two reasons: + // 1. The cache is not persisted. + // 2. We use an LRU cache to avoid extra garbage collection work. This + // means that some entries may be recycled before a pod has been + // deleted. + reasonCache stringCache } // Ensures DockerManager implements ConatinerRunner. var _ kubecontainer.ContainerRunner = new(DockerManager) func NewDockerManager(client DockerInterface, recorder record.EventRecorder, podInfraContainerImage string) *DockerManager { - return &DockerManager{client: client, recorder: recorder, PodInfraContainerImage: podInfraContainerImage} + reasonCache := stringCache{cache: lru.New(maxReasonCacheEntries)} + return &DockerManager{ + client: client, + recorder: recorder, + PodInfraContainerImage: podInfraContainerImage, + reasonCache: reasonCache} +} + +// A cache which stores strings keyed by _. +type stringCache struct { + lock sync.RWMutex + cache *lru.Cache +} + +func (self *stringCache) composeKey(uid types.UID, name string) string { + return fmt.Sprintf("%s_%s", uid, name) +} + +func (self *stringCache) Add(uid types.UID, name string, value string) { + self.lock.Lock() + defer self.lock.Unlock() + self.cache.Add(self.composeKey(uid, name), value) +} + +func (self *stringCache) Remove(uid types.UID, name string) { + self.lock.Lock() + defer self.lock.Unlock() + self.cache.Remove(self.composeKey(uid, name)) +} + +func (self *stringCache) Get(uid types.UID, name string) (string, bool) { + self.lock.RLock() + defer self.lock.RUnlock() + value, ok := self.cache.Get(self.composeKey(uid, name)) + if ok { + return value.(string), ok + } else { + return "", ok + } } // GetKubeletDockerContainerLogs returns logs of a specific container. By @@ -118,7 +172,6 @@ func (self *DockerManager) inspectContainer(dockerID, containerName, tPath strin ContainerID: DockerPrefix + dockerID, } - waiting := true if inspectResult.State.Running { result.status.State.Running = &api.ContainerStateRunning{ StartedAt: util.NewTime(inspectResult.State.StartedAt), @@ -126,7 +179,6 @@ func (self *DockerManager) inspectContainer(dockerID, containerName, tPath strin if containerName == PodInfraContainerName && inspectResult.NetworkSettings != nil { result.ip = inspectResult.NetworkSettings.IPAddress } - waiting = false } else if !inspectResult.State.FinishedAt.IsZero() { reason := "" // Note: An application might handle OOMKilled gracefully. @@ -155,13 +207,8 @@ func (self *DockerManager) inspectContainer(dockerID, containerName, tPath strin } } } - waiting = false - } - - if waiting { + } else { // TODO(dchen1107): Separate issue docker/docker#8294 was filed - // TODO(dchen1107): Need to figure out why we are still waiting - // Check any issue to run container result.status.State.Waiting = &api.ContainerStateWaiting{ Reason: ErrContainerCannotRun.Error(), } @@ -233,31 +280,27 @@ func (self *DockerManager) GetPodStatus(pod *api.Pod) (*api.PodStatus, error) { return nil, result.err } - // Add user container information - if dockerContainerName == PodInfraContainerName && - result.status.State.Running != nil { + if dockerContainerName == PodInfraContainerName { // Found network container - podStatus.PodIP = result.ip + if result.status.State.Running != nil { + podStatus.PodIP = result.ip + } } else { + // Add user container information. statuses[dockerContainerName] = result.status } } - if len(statuses) == 0 && podStatus.PodIP == "" { - return nil, ErrNoContainersInPod - } - - // Not all containers expected are created, check if there are - // image related issues - if len(statuses) < len(manifest.Containers) { + for _, container := range manifest.Containers { var containerStatus api.ContainerStatus - for _, container := range manifest.Containers { - if _, found := statuses[container.Name]; found { - continue - } - + if status, found := statuses[container.Name]; found { + containerStatus = status + } else { + // The container has not been created yet. Check image is ready on + // the node or not. + // TODO: If we integrate DockerPuller into DockerManager, we can + // record the pull failure and eliminate the image checking below. image := container.Image - // Check image is ready on the node or not // TODO(dchen1107): docker/docker/issues/8365 to figure out if the image exists _, err := self.client.InspectImage(image) if err == nil { @@ -268,14 +311,15 @@ func (self *DockerManager) GetPodStatus(pod *api.Pod) (*api.PodStatus, error) { containerStatus.State.Waiting = &api.ContainerStateWaiting{ Reason: fmt.Sprintf("Image: %s is not ready on the node", image), } - } else { - containerStatus.State.Waiting = &api.ContainerStateWaiting{ - Reason: "", - } } - - statuses[container.Name] = containerStatus } + if containerStatus.State.Waiting != nil { + // For containers in the waiting state, fill in a specific reason if it is recorded. + if reason, ok := self.reasonCache.Get(uid, container.Name); ok { + containerStatus.State.Waiting.Reason = reason + } + } + statuses[container.Name] = containerStatus } podStatus.ContainerStatuses = make([]api.ContainerStatus, 0) @@ -304,6 +348,19 @@ func (self *DockerManager) GetRunningContainers(ids []string) ([]*docker.Contain } func (self *DockerManager) RunContainer(pod *api.Pod, container *api.Container, opts *kubecontainer.RunContainerOptions) (string, error) { + dockerID, err := self.runContainer(pod, container, opts) + if err != nil { + errString := err.Error() + if errString != "" { + self.reasonCache.Add(pod.UID, container.Name, errString) + } else { + self.reasonCache.Remove(pod.UID, container.Name) + } + } + return dockerID, err +} + +func (self *DockerManager) runContainer(pod *api.Pod, container *api.Container, opts *kubecontainer.RunContainerOptions) (string, error) { ref, err := kubecontainer.GenerateContainerRef(pod, container) if err != nil { glog.Errorf("Couldn't make a ref to pod %v, container %v: '%v'", pod.Name, container.Name, err) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index ebeb056488e..49f7a3eebf7 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -1698,18 +1698,16 @@ func (kl *Kubelet) validateContainerStatus(podStatus *api.PodStatus, containerNa func (kl *Kubelet) GetKubeletContainerLogs(podFullName, containerName, tail string, follow bool, stdout, stderr io.Writer) error { podStatus, err := kl.GetPodStatus(podFullName) if err != nil { - if err == dockertools.ErrNoContainersInPod { - return fmt.Errorf("pod %q not found\n", podFullName) - } else { - return fmt.Errorf("failed to get status for pod %q - %v", podFullName, err) - } + return fmt.Errorf("failed to get status for pod %q - %v", podFullName, err) } - if err := kl.validatePodPhase(&podStatus); err != nil { + // No log is available if pod is not in a "known" phase (e.g. Unknown). return err } dockerContainerID, err := kl.validateContainerStatus(&podStatus, containerName) if err != nil { + // No log is available if the container status is missing or is in the + // waiting state. return err } return kl.containerManager.GetKubeletDockerContainerLogs(dockerContainerID, tail, follow, stdout, stderr) diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index ed81930ae9c..ce8219f5df1 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -69,7 +69,7 @@ type TestKubelet struct { } func newTestKubelet(t *testing.T) *TestKubelet { - fakeDocker := &dockertools.FakeDockerClient{RemovedImages: util.StringSet{}} + fakeDocker := &dockertools.FakeDockerClient{Errors: make(map[string]error), RemovedImages: util.StringSet{}} fakeDockerCache := dockertools.NewFakeDockerCache(fakeDocker) fakeRecorder := &record.FakeRecorder{} fakeKubeClient := &testclient.Fake{} @@ -366,7 +366,7 @@ func TestKillContainerWithError(t *testing.T) { }, } fakeDocker := &dockertools.FakeDockerClient{ - Err: fmt.Errorf("sample error"), + Errors: make(map[string]error), ContainerList: append([]docker.APIContainers{}, containers...), } testKubelet := newTestKubelet(t) @@ -376,6 +376,7 @@ func TestKillContainerWithError(t *testing.T) { } kubelet.dockerClient = fakeDocker c := apiContainerToContainer(fakeDocker.ContainerList[0]) + fakeDocker.Errors["stop"] = fmt.Errorf("sample error") err := kubelet.killContainer(&c) if err == nil { t.Errorf("expected error, found nil") @@ -512,7 +513,7 @@ func TestSyncPodsWithTerminationLog(t *testing.T) { } waitGroup.Wait() verifyCalls(t, fakeDocker, []string{ - "list", "list", "list", "create", "start", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"}) + "list", "list", "list", "inspect_image", "create", "start", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"}) fakeDocker.Lock() parts := strings.Split(fakeDocker.Container.HostConfig.Binds[0], ":") @@ -564,7 +565,7 @@ func TestSyncPodsCreatesNetAndContainer(t *testing.T) { waitGroup.Wait() verifyCalls(t, fakeDocker, []string{ - "list", "list", "list", "create", "start", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"}) + "list", "list", "list", "inspect_image", "create", "start", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"}) fakeDocker.Lock() @@ -619,7 +620,7 @@ func TestSyncPodsCreatesNetAndContainerPullsImage(t *testing.T) { waitGroup.Wait() verifyCalls(t, fakeDocker, []string{ - "list", "list", "list", "create", "start", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"}) + "list", "list", "list", "inspect_image", "create", "start", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"}) fakeDocker.Lock() @@ -671,7 +672,7 @@ func TestSyncPodsWithPodInfraCreatesContainer(t *testing.T) { waitGroup.Wait() verifyCalls(t, fakeDocker, []string{ - "list", "list", "list", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"}) + "list", "list", "list", "inspect_container", "inspect_image", "create", "start", "list", "inspect_container", "inspect_container"}) fakeDocker.Lock() if len(fakeDocker.Created) != 1 || @@ -730,7 +731,7 @@ func TestSyncPodsWithPodInfraCreatesContainerCallsHandler(t *testing.T) { waitGroup.Wait() verifyCalls(t, fakeDocker, []string{ - "list", "list", "list", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"}) + "list", "list", "list", "inspect_container", "inspect_image", "create", "start", "list", "inspect_container", "inspect_container"}) fakeDocker.Lock() if len(fakeDocker.Created) != 1 || @@ -948,7 +949,7 @@ func TestSyncPodDeletesDuplicate(t *testing.T) { t.Errorf("unexpected error: %v", err) } - verifyCalls(t, fakeDocker, []string{"list", "stop", "list"}) + verifyCalls(t, fakeDocker, []string{"list", "inspect_image", "stop", "list", "inspect_image"}) // Expect one of the duplicates to be killed. if len(fakeDocker.Stopped) != 1 || (fakeDocker.Stopped[0] != "1234" && fakeDocker.Stopped[0] != "4567") { t.Errorf("Wrong containers were stopped: %v", fakeDocker.Stopped) @@ -990,8 +991,8 @@ func TestSyncPodBadHash(t *testing.T) { t.Errorf("unexpected error: %v", err) } - //verifyCalls(t, fakeDocker, []string{"list", "stop", "list", "create", "start", "stop", "create", "start", "inspect_container"}) - verifyCalls(t, fakeDocker, []string{"list", "stop", "stop", "create", "start", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"}) + verifyCalls(t, fakeDocker, []string{"list", "inspect_image", "stop", "stop", "create", "start", + "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"}) // A map interation is used to delete containers, so must not depend on // order here. @@ -1045,7 +1046,7 @@ func TestSyncPodUnhealthy(t *testing.T) { t.Errorf("unexpected error: %v", err) } - verifyCalls(t, fakeDocker, []string{"list", "stop", "create", "start", "list", "inspect_container"}) + verifyCalls(t, fakeDocker, []string{"list", "inspect_image", "stop", "create", "start", "list", "inspect_container"}) // A map interation is used to delete containers, so must not depend on // order here. @@ -1638,7 +1639,7 @@ func TestSyncPodEventHandlerFails(t *testing.T) { t.Errorf("unexpected error: %v", err) } - verifyCalls(t, fakeDocker, []string{"list", "create", "start", "stop", "list"}) + verifyCalls(t, fakeDocker, []string{"list", "inspect_image", "create", "start", "stop", "list", "inspect_image"}) if len(fakeDocker.Stopped) != 1 { t.Errorf("Wrong containers were stopped: %v", fakeDocker.Stopped) @@ -3792,3 +3793,44 @@ func TestGetPodStatusWithLastTermination(t *testing.T) { } } } + +func TestGetPodCreationFailureReason(t *testing.T) { + testKubelet := newTestKubelet(t) + kubelet := testKubelet.kubelet + fakeDocker := testKubelet.fakeDocker + failureReason := "creation failure" + fakeDocker.Errors["create"] = fmt.Errorf("%s", failureReason) + fakeDocker.ContainerList = []docker.APIContainers{} + pod := api.Pod{ + ObjectMeta: api.ObjectMeta{ + UID: "12345678", + Name: "bar", + Namespace: "new", + }, + Spec: api.PodSpec{ + Containers: []api.Container{ + {Name: "foo"}, + }, + }, + } + pods := []api.Pod{pod} + kubelet.podManager.SetPods(pods) + _, err := kubelet.runContainer(&pod, &pod.Spec.Containers[0], make(map[string]volume.Volume), "", "") + if err == nil { + t.Errorf("expected error, found nil") + } + status, err := kubelet.GetPodStatus(kubecontainer.GetPodFullName(&pod)) + if err != nil { + t.Errorf("unexpected error %v", err) + } + if len(status.ContainerStatuses) < 1 { + t.Errorf("expected 1 container status, got %d", len(status.ContainerStatuses)) + } else { + state := status.ContainerStatuses[0].State + if state.Waiting == nil { + t.Errorf("expected waiting state, got %#v", state) + } else if state.Waiting.Reason != failureReason { + t.Errorf("expected reason %q, got %q", state.Waiting.Reason) + } + } +}