kubeadm: Make etcd member removal idempotent

If the etcd member is not found, then it has already been removed, and
kubeadm reset should immediately complete the 'remove-etcd-member'
phase. Previously, the phase would complete only once the
exponential-backoff retry expired, up to 3 minutes duration.

This commit also fixes a semantic error in etcd.GetMemberID. Previously,
the function returned 0 if no member was found, but 0 is not a valid
member ID.

cmd/kubeadm/app/phases/etcd/local.go

@@ -118,6 +118,10 @@ func RemoveStackedEtcdMemberFromCluster(client clientset.Interface, cfg *kubeadm
 	klog.V(2).Infof("[etcd] get the member id from peer: %s", etcdPeerAddress)
 	id, err := etcdClient.GetMemberID(etcdPeerAddress)
 	if err != nil {
+		if errors.Is(etcdutil.ErrNoMemberIDForPeerURL, err) {
+			klog.V(5).Infof("[etcd] member was already removed, because no member id exists for peer %s", etcdPeerAddress)
+			return nil
+		}
 		return err
 	}
 

cmd/kubeadm/app/util/etcd/etcd.go

@@ -53,6 +53,8 @@ var etcdBackoff = wait.Backoff{
 	Jitter:   0.1,
 }
 
+var ErrNoMemberIDForPeerURL = errors.New("no member id found for peer URL")
+
 // ClusterInterrogator is an interface to get etcd cluster related information
 type ClusterInterrogator interface {
 	CheckClusterHealth() error
@@ -310,7 +312,7 @@ func (c *Client) GetMemberID(peerURL string) (uint64, error) {
 			return member.GetID(), nil
 		}
 	}
-	return 0, nil
+	return 0, ErrNoMemberIDForPeerURL
 }
 
 // ListMembers returns the member list.
@@ -346,6 +348,10 @@ func (c *Client) RemoveMember(id uint64) ([]Member, error) {
 		if err == nil {
 			return true, nil
 		}
+		if errors.Is(rpctypes.ErrMemberNotFound, err) {
+			klog.V(5).Infof("Member was already removed, because member %016x was not found", id)
+			return true, nil
+		}
 		klog.V(5).Infof("Failed to remove etcd member: %v", err)
 		lastError = err
 		return false, nil

cmd/kubeadm/app/util/etcd/etcd_test.go

@@ -423,7 +423,7 @@ func TestClient_GetMemberID(t *testing.T) {
 		fields  fields
 		args    args
 		want    uint64
-		wantErr bool
+		wantErr error
 	}{
 		{
 			name: "member ID found",
@@ -447,7 +447,7 @@ func TestClient_GetMemberID(t *testing.T) {
 			args: args{
 				peerURL: "https://member1:2380",
 			},
-			wantErr: false,
+			wantErr: nil,
 			want:    1,
 		},
 		{
@@ -472,7 +472,7 @@ func TestClient_GetMemberID(t *testing.T) {
 			args: args{
 				peerURL: "https://member2:2380",
 			},
-			wantErr: false,
+			wantErr: ErrNoMemberIDForPeerURL,
 			want:    0,
 		},
 	}
@@ -484,7 +484,7 @@ func TestClient_GetMemberID(t *testing.T) {
 			}
 
 			got, err := c.GetMemberID(tt.args.peerURL)
-			if (err != nil) != tt.wantErr {
+			if !errors.Is(tt.wantErr, err) {
 				t.Errorf("Client.GetMemberID() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}