github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-22 03:11:40 +00:00
Code Issues Projects Releases Wiki Activity

statefulset: handle API error on pod deletion

Browse Source 
when new revision is being rolled out
This commit is contained in:
Filip Křepinský 2022-11-10 22:29:18 +01:00
parent 3649eeae9c
commit ec0b200f3d
2 changed files with 75 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/controller/statefulset/stateful_set_control.go
View File

@ -573,7 +573,11 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet(
if getPodRevision(replicas[target]) != updateRevision.Name && !isTerminating(replicas[target]) {
klog.V(2).InfoS("Pod of StatefulSet is terminating for update",
"statefulSet", klog.KObj(set), "pod", klog.KObj(replicas[target]))
err := ssc.podControl.DeleteStatefulPod(set, replicas[target])
if err := ssc.podControl.DeleteStatefulPod(set, replicas[target]); err != nil {
if !errors.IsNotFound(err) {
return &status, err
}
}
status.CurrentReplicas--
return &status, err
}

70
pkg/controller/statefulset/stateful_set_control_test.go
View File

@ -173,6 +173,7 @@ func TestStatefulSetControl(t *testing.T) {
{UpdatePodFailure, simpleSetFn},
{UpdateSetStatusFailure, simpleSetFn},
{PodRecreateDeleteFailure, simpleSetFn},
{NewRevisionDeletePodFailure, simpleSetFn},
}
for _, testCase := range testCases {
@ -559,6 +560,75 @@ func PodRecreateDeleteFailure(t *testing.T, set *apps.StatefulSet, invariants in
}
}
func NewRevisionDeletePodFailure(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) {
client := fake.NewSimpleClientset(set)
om, _, ssc := setupController(client)
if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil {
t.Errorf("Failed to turn up StatefulSet : %s", err)
}
var err error
set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name)
if err != nil {
t.Fatalf("Error getting updated StatefulSet: %v", err)
}
if set.Status.Replicas != 3 {
t.Error("Failed to scale StatefulSet to 3 replicas")
}
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
if err != nil {
t.Error(err)
}
pods, err := om.podsLister.Pods(set.Namespace).List(selector)
if err != nil {
t.Error(err)
}
// trigger a new revision
updateSet := set.DeepCopy()
updateSet.Spec.Template.Spec.Containers[0].Image = "nginx-new"
if err := om.setsIndexer.Update(updateSet); err != nil {
t.Error("Failed to update StatefulSet")
}
set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name)
if err != nil {
t.Fatalf("Error getting updated StatefulSet: %v", err)
}
// delete fails
om.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0)
_, err = ssc.UpdateStatefulSet(context.TODO(), set, pods)
if err == nil {
t.Error("Expected err in update StatefulSet when deleting a pod")
}
set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name)
if err != nil {
t.Fatalf("Error getting updated StatefulSet: %v", err)
}
if err := invariants(set, om); err != nil {
t.Error(err)
}
if set.Status.CurrentReplicas != 3 {
t.Fatalf("Failed pod deletion should not update CurrentReplicas: want 3, got %d", set.Status.CurrentReplicas)
}
if set.Status.CurrentRevision == set.Status.UpdateRevision {
t.Error("Failed to create new revision")
}
// delete works
om.SetDeleteStatefulPodError(nil, 0)
status, err := ssc.UpdateStatefulSet(context.TODO(), set, pods)
if err != nil {
t.Fatalf("Unexpected err in update StatefulSet: %v", err)
}
if status.CurrentReplicas != 2 {
t.Fatalf("Pod deletion should update CurrentReplicas: want 2, got %d", status.CurrentReplicas)
}
if err := invariants(set, om); err != nil {
t.Error(err)
}
}
func emptyInvariants(set *apps.StatefulSet, om *fakeObjectManager) error {
return nil
}