mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-01-06 07:57:35 +00:00
Send recycle events from pod to pv.
This allows users to diagnose what's wrong with recycler. Recycler pods are
started automatically with a cryptic name and they are deleted immediately
when they finish.
kubectl describe pods will show:
FirstSeen LastSeen Count From SubobjectPath Type Reason Message
--------- -------- ----- ---- ------------- -------- ------ -------
59m 59m 1 {persistentvolume-controller } Warning RecyclerPod Recycler pod: Unable to mount volumes for pod "recycler-for-nfs_default(5421800e-347b-11e6-a79b-3c970e965218)": timeout expired waiting for volumes to attach/mount for pod "recycler-for-nfs"/"default". list of unattached/unmounted volumes=[vol]
53m 53m 1 {persistentvolume-controller } Warning RecyclerPod Recycler pod: Unable to mount volumes for pod "recycler-for-nfs_default(3c9809e5-347c-11e6-a79b-3c970e965218)": timeout expired waiting for volumes to attach/mount for pod "recycler-for-nfs"/"default". list of unattached/unmounted volumes=[vol]
46m 46m 1 {persistentvolume-controller } Warning RecyclerPod Recycler pod: Unable to mount volumes for pod "recycler-for-nfs_default(250dd2a2-347d-11e6-a79b-3c970e965218)": timeout expired waiting for volumes to attach/mount for pod "recycler-for-nfs"/"default". list of unattached/unmounted volumes=[vol]
40m 40m 1 {persistentvolume-controller } Warning RecyclerPod Recycler pod: Unable to mount volumes for pod "recycler-for-nfs_default(0d84ea33-347e-11e6-a79b-3c970e965218)": timeout expired waiting for volumes to attach/mount for pod "recycler-for-nfs"/"default". list of unattached/unmounted volumes=[vol]
33m 33m 1 {persistentvolume-controller } Warning RecyclerPod Recycler pod: Unable to mount volumes for pod "recycler-for-nfs_default(f5fb63bf-347e-11e6-a79b-3c970e965218)": timeout expired waiting for volumes to attach/mount for pod "recycler-for-nfs"/"default". list of unattached/unmounted volumes=[vol]
27m 27m 1 {persistentvolume-controller } Warning RecyclerPod Recycler pod: Unable to mount volumes for pod "recycler-for-nfs_default(de7128fd-347f-11e6-a79b-3c970e965218)": timeout expired waiting for volumes to attach/mount for pod "recycler-for-nfs"/"default". list of unattached/unmounted volumes=[vol]
1h 3m 75 {persistentvolume-controller } Normal RecyclerPod Recycler pod: Successfully assigned recycler-for-nfs to 127.0.0.1
1h 3m 76 {persistentvolume-controller } Normal RecyclerPod Recycler pod: Pod was active on the node longer than specified deadline
1h 1m 12 {persistentvolume-controller } Warning RecyclerPod Recycler pod: Error syncing pod, skipping: timeout expired waiting for volumes to attach/mount for pod "recycler-for-nfs"/"default". list of unattached/unmounted volumes=[vol]
20m 1m 4 {persistentvolume-controller } Warning RecyclerPod (events with common reason combined)
These steps were necessary:
- added event watcher to volume.RecycleVolumeByWatchingPodUntilCompletion
- pass all these events through volume plugins to volume controller
- rework volume.RecycleVolumeByWatchingPodUntilCompletion unit tests to a table
(too much copy-paste)
- fix all unit tests along the way
This commit is contained in:
Jan Safranek
@@ -24,106 +24,195 @@ import (
|
||||
"k8s.io/kubernetes/pkg/api"
|
||||
"k8s.io/kubernetes/pkg/api/errors"
|
||||
"k8s.io/kubernetes/pkg/api/resource"
|
||||
"k8s.io/kubernetes/pkg/watch"
|
||||
)
|
||||
|
||||
func TestRecyclerSuccess(t *testing.T) {
|
||||
client := &mockRecyclerClient{}
|
||||
recycler := &api.Pod{
|
||||
ObjectMeta: api.ObjectMeta{
|
||||
Namespace: api.NamespaceDefault,
|
||||
},
|
||||
Status: api.PodStatus{
|
||||
Phase: api.PodSucceeded,
|
||||
},
|
||||
}
|
||||
type testcase struct {
|
||||
// Input of the test
|
||||
name string
|
||||
existingPod *api.Pod
|
||||
createPod *api.Pod
|
||||
// eventSequence is list of events that are simulated during recycling. It
|
||||
// can be either event generated by a recycler pod or a state change of
|
||||
// the pod. (see newPodEvent and newEvent below).
|
||||
eventSequence []watch.Event
|
||||
|
||||
err := internalRecycleVolumeByWatchingPodUntilCompletion("pv-name", recycler, client)
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error watching recycler pod: %+v", err)
|
||||
}
|
||||
if !client.deletedCalled {
|
||||
t.Errorf("Expected deferred client.Delete to be called on recycler pod")
|
||||
// Expected output.
|
||||
// expectedEvents is list of events that were sent to the volume that was
|
||||
// recycled.
|
||||
expectedEvents []mockEvent
|
||||
expectedError string
|
||||
}
|
||||
|
||||
func newPodEvent(eventtype watch.EventType, name string, phase api.PodPhase, message string) watch.Event {
|
||||
return watch.Event{
|
||||
Type: eventtype,
|
||||
Object: newPod(name, phase, message),
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecyclerFailure(t *testing.T) {
|
||||
client := &mockRecyclerClient{}
|
||||
recycler := &api.Pod{
|
||||
ObjectMeta: api.ObjectMeta{
|
||||
Namespace: api.NamespaceDefault,
|
||||
func newEvent(eventtype, message string) watch.Event {
|
||||
return watch.Event{
|
||||
Type: watch.Added,
|
||||
Object: &api.Event{
|
||||
ObjectMeta: api.ObjectMeta{
|
||||
Namespace: api.NamespaceDefault,
|
||||
},
|
||||
Reason: "MockEvent",
|
||||
Message: message,
|
||||
Type: eventtype,
|
||||
},
|
||||
Status: api.PodStatus{
|
||||
Phase: api.PodFailed,
|
||||
Message: "foo",
|
||||
},
|
||||
}
|
||||
|
||||
err := internalRecycleVolumeByWatchingPodUntilCompletion("pv-name", recycler, client)
|
||||
if err == nil {
|
||||
t.Fatalf("Expected pod failure but got nil error returned")
|
||||
}
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "foo") {
|
||||
t.Errorf("Expected pod.Status.Message %s but got %s", recycler.Status.Message, err)
|
||||
}
|
||||
}
|
||||
if !client.deletedCalled {
|
||||
t.Errorf("Expected deferred client.Delete to be called on recycler pod")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecyclerAlreadyExists(t *testing.T) {
|
||||
// Test that internalRecycleVolumeByWatchingPodUntilCompletion does not
|
||||
// start a new recycler when an old one is already running.
|
||||
|
||||
// Old recycler is running and fails with "foo" error message
|
||||
oldRecycler := &api.Pod{
|
||||
func newPod(name string, phase api.PodPhase, message string) *api.Pod {
|
||||
return &api.Pod{
|
||||
ObjectMeta: api.ObjectMeta{
|
||||
Name: "recycler-test",
|
||||
Namespace: api.NamespaceDefault,
|
||||
Name: name,
|
||||
},
|
||||
Status: api.PodStatus{
|
||||
Phase: api.PodFailed,
|
||||
Message: "foo",
|
||||
Phase: phase,
|
||||
Message: message,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecyclerPod(t *testing.T) {
|
||||
tests := []testcase{
|
||||
{
|
||||
// Test recycler success with some events
|
||||
name: "RecyclerSuccess",
|
||||
createPod: newPod("podRecyclerSuccess", api.PodPending, ""),
|
||||
eventSequence: []watch.Event{
|
||||
// Pod gets Running and Succeeded
|
||||
newPodEvent(watch.Added, "podRecyclerSuccess", api.PodPending, ""),
|
||||
newEvent(api.EventTypeNormal, "Successfully assigned recycler-for-podRecyclerSuccess to 127.0.0.1"),
|
||||
newEvent(api.EventTypeNormal, "pulling image \"gcr.io/google_containers/busybox\""),
|
||||
newEvent(api.EventTypeNormal, "Successfully pulled image \"gcr.io/google_containers/busybox\""),
|
||||
newEvent(api.EventTypeNormal, "Created container with docker id 83d929aeac82"),
|
||||
newEvent(api.EventTypeNormal, "Started container with docker id 83d929aeac82"),
|
||||
newPodEvent(watch.Modified, "podRecyclerSuccess", api.PodRunning, ""),
|
||||
newPodEvent(watch.Modified, "podRecyclerSuccess", api.PodSucceeded, ""),
|
||||
},
|
||||
expectedEvents: []mockEvent{
|
||||
{api.EventTypeNormal, "Successfully assigned recycler-for-podRecyclerSuccess to 127.0.0.1"},
|
||||
{api.EventTypeNormal, "pulling image \"gcr.io/google_containers/busybox\""},
|
||||
{api.EventTypeNormal, "Successfully pulled image \"gcr.io/google_containers/busybox\""},
|
||||
{api.EventTypeNormal, "Created container with docker id 83d929aeac82"},
|
||||
{api.EventTypeNormal, "Started container with docker id 83d929aeac82"},
|
||||
},
|
||||
expectedError: "",
|
||||
},
|
||||
{
|
||||
// Test recycler failure with some events
|
||||
name: "RecyclerFailure",
|
||||
createPod: newPod("podRecyclerFailure", api.PodPending, ""),
|
||||
eventSequence: []watch.Event{
|
||||
// Pod gets Running and Succeeded
|
||||
newPodEvent(watch.Added, "podRecyclerFailure", api.PodPending, ""),
|
||||
newEvent(api.EventTypeNormal, "Successfully assigned recycler-for-podRecyclerFailure to 127.0.0.1"),
|
||||
newEvent(api.EventTypeWarning, "Unable to mount volumes for pod \"recycler-for-podRecyclerFailure_default(3c9809e5-347c-11e6-a79b-3c970e965218)\": timeout expired waiting for volumes to attach/mount"),
|
||||
newEvent(api.EventTypeWarning, "Error syncing pod, skipping: timeout expired waiting for volumes to attach/mount for pod \"recycler-for-podRecyclerFailure\"/\"default\". list of unattached/unmounted"),
|
||||
newPodEvent(watch.Modified, "podRecyclerFailure", api.PodRunning, ""),
|
||||
newPodEvent(watch.Modified, "podRecyclerFailure", api.PodFailed, "Pod was active on the node longer than specified deadline"),
|
||||
},
|
||||
expectedEvents: []mockEvent{
|
||||
{api.EventTypeNormal, "Successfully assigned recycler-for-podRecyclerFailure to 127.0.0.1"},
|
||||
{api.EventTypeWarning, "Unable to mount volumes for pod \"recycler-for-podRecyclerFailure_default(3c9809e5-347c-11e6-a79b-3c970e965218)\": timeout expired waiting for volumes to attach/mount"},
|
||||
{api.EventTypeWarning, "Error syncing pod, skipping: timeout expired waiting for volumes to attach/mount for pod \"recycler-for-podRecyclerFailure\"/\"default\". list of unattached/unmounted"},
|
||||
},
|
||||
expectedError: "Pod was active on the node longer than specified deadline",
|
||||
},
|
||||
{
|
||||
// Recycler pod gets deleted
|
||||
name: "RecyclerDeleted",
|
||||
createPod: newPod("podRecyclerDeleted", api.PodPending, ""),
|
||||
eventSequence: []watch.Event{
|
||||
// Pod gets Running and Succeeded
|
||||
newPodEvent(watch.Added, "podRecyclerDeleted", api.PodPending, ""),
|
||||
newEvent(api.EventTypeNormal, "Successfully assigned recycler-for-podRecyclerDeleted to 127.0.0.1"),
|
||||
newPodEvent(watch.Deleted, "podRecyclerDeleted", api.PodPending, ""),
|
||||
},
|
||||
expectedEvents: []mockEvent{
|
||||
{api.EventTypeNormal, "Successfully assigned recycler-for-podRecyclerDeleted to 127.0.0.1"},
|
||||
},
|
||||
expectedError: "recycler pod was deleted",
|
||||
},
|
||||
{
|
||||
// Another recycler pod is already running
|
||||
name: "RecyclerRunning",
|
||||
existingPod: newPod("podOldRecycler", api.PodRunning, ""),
|
||||
createPod: newPod("podNewRecycler", api.PodFailed, "mock message"),
|
||||
eventSequence: []watch.Event{
|
||||
// Old pod succeeds
|
||||
newPodEvent(watch.Modified, "podOldRecycler", api.PodSucceeded, ""),
|
||||
},
|
||||
// No error = old pod succeeded. If the new pod was used, there
|
||||
// would be error with "mock message".
|
||||
expectedError: "",
|
||||
},
|
||||
{
|
||||
// Another recycler pod is already running and fails
|
||||
name: "FailedRecyclerRunning",
|
||||
existingPod: newPod("podOldRecycler", api.PodRunning, ""),
|
||||
createPod: newPod("podNewRecycler", api.PodFailed, "mock message"),
|
||||
eventSequence: []watch.Event{
|
||||
// Old pod failure
|
||||
newPodEvent(watch.Modified, "podOldRecycler", api.PodFailed, "Pod was active on the node longer than specified deadline"),
|
||||
},
|
||||
// If the new pod was used, there would be error with "mock message".
|
||||
expectedError: "Pod was active on the node longer than specified deadline",
|
||||
},
|
||||
}
|
||||
|
||||
// New recycler _would_ succeed if it was run
|
||||
newRecycler := &api.Pod{
|
||||
ObjectMeta: api.ObjectMeta{
|
||||
Name: "recycler-test",
|
||||
Namespace: api.NamespaceDefault,
|
||||
},
|
||||
Status: api.PodStatus{
|
||||
Phase: api.PodSucceeded,
|
||||
Message: "bar",
|
||||
},
|
||||
}
|
||||
|
||||
client := &mockRecyclerClient{
|
||||
pod: oldRecycler,
|
||||
}
|
||||
|
||||
err := internalRecycleVolumeByWatchingPodUntilCompletion("pv-name", newRecycler, client)
|
||||
if err == nil {
|
||||
t.Fatalf("Expected pod failure but got nil error returned")
|
||||
}
|
||||
|
||||
// Check the recycler failed with "foo" error message, i.e. it was the
|
||||
// old recycler that finished and not the new one.
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "foo") {
|
||||
t.Errorf("Expected pod.Status.Message %s but got %s", oldRecycler.Status.Message, err)
|
||||
for _, test := range tests {
|
||||
t.Logf("Test %q", test.name)
|
||||
client := &mockRecyclerClient{
|
||||
events: test.eventSequence,
|
||||
pod: test.existingPod,
|
||||
}
|
||||
err := internalRecycleVolumeByWatchingPodUntilCompletion(test.createPod.Name, test.createPod, client)
|
||||
receivedError := ""
|
||||
if err != nil {
|
||||
receivedError = err.Error()
|
||||
}
|
||||
if receivedError != test.expectedError {
|
||||
t.Errorf("Test %q failed, expected error %q, got %q", test.name, test.expectedError, receivedError)
|
||||
continue
|
||||
}
|
||||
if !client.deletedCalled {
|
||||
t.Errorf("Test %q failed, expected deferred client.Delete to be called on recycler pod", test.name)
|
||||
continue
|
||||
}
|
||||
for i, expectedEvent := range test.expectedEvents {
|
||||
if len(client.receivedEvents) <= i {
|
||||
t.Errorf("Test %q failed, expected event %d: %q not received", test.name, i, expectedEvent.message)
|
||||
continue
|
||||
}
|
||||
receivedEvent := client.receivedEvents[i]
|
||||
if expectedEvent.eventtype != receivedEvent.eventtype {
|
||||
t.Errorf("Test %q failed, event %d does not match: expected eventtype %q, got %q", test.name, i, expectedEvent.eventtype, receivedEvent.eventtype)
|
||||
}
|
||||
if expectedEvent.message != receivedEvent.message {
|
||||
t.Errorf("Test %q failed, event %d does not match: expected message %q, got %q", test.name, i, expectedEvent.message, receivedEvent.message)
|
||||
}
|
||||
}
|
||||
for i := len(test.expectedEvents); i < len(client.receivedEvents); i++ {
|
||||
t.Errorf("Test %q failed, unexpected event received: %s, %q", test.name, client.receivedEvents[i].eventtype, client.receivedEvents[i].message)
|
||||
}
|
||||
}
|
||||
if !client.deletedCalled {
|
||||
t.Errorf("Expected deferred client.Delete to be called on recycler pod")
|
||||
}
|
||||
}
|
||||
|
||||
type mockRecyclerClient struct {
|
||||
pod *api.Pod
|
||||
deletedCalled bool
|
||||
pod *api.Pod
|
||||
deletedCalled bool
|
||||
receivedEvents []mockEvent
|
||||
events []watch.Event
|
||||
}
|
||||
|
||||
type mockEvent struct {
|
||||
eventtype, message string
|
||||
}
|
||||
|
||||
func (c *mockRecyclerClient) CreatePod(pod *api.Pod) (*api.Pod, error) {
|
||||
@@ -148,10 +237,18 @@ func (c *mockRecyclerClient) DeletePod(name, namespace string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *mockRecyclerClient) WatchPod(name, namespace string, stopChannel chan struct{}) func() *api.Pod {
|
||||
return func() *api.Pod {
|
||||
return c.pod
|
||||
}
|
||||
func (c *mockRecyclerClient) WatchPod(name, namespace string, stopChannel chan struct{}) (<-chan watch.Event, error) {
|
||||
eventCh := make(chan watch.Event, 0)
|
||||
go func() {
|
||||
for _, e := range c.events {
|
||||
eventCh <- e
|
||||
}
|
||||
}()
|
||||
return eventCh, nil
|
||||
}
|
||||
|
||||
func (c *mockRecyclerClient) Event(eventtype, message string) {
|
||||
c.receivedEvents = append(c.receivedEvents, mockEvent{eventtype, message})
|
||||
}
|
||||
|
||||
func TestCalculateTimeoutForVolume(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user