Merge pull request #34648 from nikhiljindal/NSCasDel

Automatic merge from submit-queue

Adding cascading deletion support to federated namespaces

Ref https://github.com/kubernetes/kubernetes/issues/33612

With this change, whenever a federated namespace is deleted with `DeleteOptions.OrphanDependents = false`, then federation namespace controller first deletes the corresponding namespaces from all underlying clusters before deleting the federated namespace.

cc @kubernetes/sig-cluster-federation @caesarxuchao


```release-note
Adding support for DeleteOptions.OrphanDependents for federated namespaces. Setting it to false while deleting a federated namespace also deletes the corresponding namespace from all registered clusters.
```

federation/pkg/federation-controller/util/deletionhelper/BUILD

@@ -0,0 +1,25 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+    "go_test",
+    "cgo_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["deletion_helper.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//pkg/api:go_default_library",
+        "//pkg/api/v1:go_default_library",
+        "//pkg/client/record:go_default_library",
+        "//pkg/runtime:go_default_library",
+        "//vendor:github.com/golang/glog",
+    ],
+)

federation/pkg/federation-controller/util/deletionhelper/deletion_helper.go

@@ -0,0 +1,171 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package to help federation controllers to delete federated resources from
+// underlying clusters when the resource is deleted from federation control
+// plane.
+package deletionhelper
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	"k8s.io/kubernetes/pkg/api"
+	api_v1 "k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/client/record"
+	"k8s.io/kubernetes/pkg/runtime"
+
+	"github.com/golang/glog"
+)
+
+const (
+	// Add this finalizer to a federation resource if the resource should be
+	// deleted from all underlying clusters before being deleted from
+	// federation control plane.
+	// This is ignored if FinalizerOrphan is also present on the resource.
+	// In that case, both finalizers are removed from the resource and the
+	// resource is deleted from federation control plane without affecting
+	// the underlying clusters.
+	FinalizerDeleteFromUnderlyingClusters string = "federation.kubernetes.io/delete-from-underlying-clusters"
+)
+
+type HasFinalizerFunc func(runtime.Object, string) bool
+type RemoveFinalizerFunc func(runtime.Object, string) (runtime.Object, error)
+type AddFinalizerFunc func(runtime.Object, string) (runtime.Object, error)
+type ObjNameFunc func(runtime.Object) string
+
+type DeletionHelper struct {
+	hasFinalizerFunc    HasFinalizerFunc
+	removeFinalizerFunc RemoveFinalizerFunc
+	addFinalizerFunc    AddFinalizerFunc
+	objNameFunc         ObjNameFunc
+	updateTimeout       time.Duration
+	eventRecorder       record.EventRecorder
+	informer            util.FederatedInformer
+	updater             util.FederatedUpdater
+}
+
+func NewDeletionHelper(
+	hasFinalizerFunc HasFinalizerFunc, removeFinalizerFunc RemoveFinalizerFunc,
+	addFinalizerFunc AddFinalizerFunc, objNameFunc ObjNameFunc,
+	updateTimeout time.Duration, eventRecorder record.EventRecorder,
+	informer util.FederatedInformer,
+	updater util.FederatedUpdater) *DeletionHelper {
+	return &DeletionHelper{
+		hasFinalizerFunc:    hasFinalizerFunc,
+		removeFinalizerFunc: removeFinalizerFunc,
+		addFinalizerFunc:    addFinalizerFunc,
+		objNameFunc:         objNameFunc,
+		updateTimeout:       updateTimeout,
+		eventRecorder:       eventRecorder,
+		informer:            informer,
+		updater:             updater,
+	}
+}
+
+// Ensures that the given object has the required finalizer to ensure that
+// objects are deleted in underlying clusters when this object is deleted
+// from federation control plane.
+// This method should be called before creating objects in underlying clusters.
+func (dh *DeletionHelper) EnsureDeleteFromUnderlyingClustersFinalizer(obj runtime.Object) (
+	runtime.Object, error) {
+	if dh.hasFinalizerFunc(obj, FinalizerDeleteFromUnderlyingClusters) {
+		return obj, nil
+	}
+	return dh.addFinalizerFunc(obj, FinalizerDeleteFromUnderlyingClusters)
+}
+
+// Deletes the resources corresponding to the given federated resource from
+// all underlying clusters, unless it has the FinalizerOrphan finalizer.
+// Removes FinalizerOrphan and FinalizerDeleteFromUnderlyingClusters finalizers
+// when done.
+// Callers are expected to keep calling this (with appropriate backoff) until
+// it succeeds.
+func (dh *DeletionHelper) HandleObjectInUnderlyingClusters(obj runtime.Object) (
+	runtime.Object, error) {
+	objName := dh.objNameFunc(obj)
+	glog.V(2).Infof("Handling deletion of federated dependents for object: %s", objName)
+	if !dh.hasFinalizerFunc(obj, FinalizerDeleteFromUnderlyingClusters) {
+		glog.V(2).Infof("obj does not have %s finalizer. Nothing to do", FinalizerDeleteFromUnderlyingClusters)
+		return obj, nil
+	}
+	hasOrphanFinalizer := dh.hasFinalizerFunc(obj, api_v1.FinalizerOrphan)
+	if hasOrphanFinalizer {
+		glog.V(3).Infof("Found finalizer orphan. Nothing to do, just remove the finalizer")
+		// If the obj has FinalizerOrphan finalizer, then we need to orphan the
+		// corresponding objects in underlying clusters.
+		// Just remove both the finalizers in that case.
+		obj, err := dh.removeFinalizerFunc(obj, api_v1.FinalizerOrphan)
+		if err != nil {
+			return obj, err
+		}
+		return dh.removeFinalizerFunc(obj, FinalizerDeleteFromUnderlyingClusters)
+	}
+
+	// Else, we need to delete the obj from all underlying clusters.
+	unreadyClusters, err := dh.informer.GetUnreadyClusters()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get a list of unready clusters: %v", err)
+	}
+	// TODO: Handle the case when cluster resource is watched after this is executed.
+	// This can happen if a namespace is deleted before its creation had been
+	// observed in all underlying clusters.
+	clusterNsObjs, err := dh.informer.GetTargetStore().GetFromAllClusters(objName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get object %s from underlying clusters: %v", objName, err)
+	}
+	operations := make([]util.FederatedOperation, 0)
+	for _, clusterNsObj := range clusterNsObjs {
+		operations = append(operations, util.FederatedOperation{
+			Type:        util.OperationTypeDelete,
+			ClusterName: clusterNsObj.ClusterName,
+			Obj:         clusterNsObj.Object.(runtime.Object),
+		})
+	}
+	err = dh.updater.UpdateWithOnError(operations, dh.updateTimeout, func(op util.FederatedOperation, operror error) {
+		objName := dh.objNameFunc(op.Obj)
+		dh.eventRecorder.Eventf(obj, api.EventTypeNormal, "DeleteInClusterFailed",
+			"Failed to delete obj %s in cluster %s: %v", objName, op.ClusterName, operror)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute updates for obj %s: %v", objName, err)
+	}
+	if len(operations) > 0 {
+		// We have deleted a bunch of resources.
+		// Wait for the store to observe all the deletions.
+		var clusterNames []string
+		for _, op := range operations {
+			clusterNames = append(clusterNames, op.ClusterName)
+		}
+		return nil, fmt.Errorf("waiting for object %s to be deleted from clusters: %s", objName, strings.Join(clusterNames, ", "))
+	}
+
+	// We have now deleted the object from all *ready* clusters.
+	// But still need to wait for clusters that are not ready to ensure that
+	// the object has been deleted from *all* clusters.
+	if len(unreadyClusters) != 0 {
+		var clusterNames []string
+		for _, cluster := range unreadyClusters {
+			clusterNames = append(clusterNames, cluster.Name)
+		}
+		return nil, fmt.Errorf("waiting for clusters %s to become ready to verify that obj %s has been deleted", strings.Join(clusterNames, ", "), objName)
+	}
+
+	// All done. Just remove the finalizer.
+	return dh.removeFinalizerFunc(obj, FinalizerDeleteFromUnderlyingClusters)
+}

federation/pkg/federation-controller/util/federated_informer.go

@@ -74,6 +74,9 @@ type FederationView interface {
 	// GetClientsetForCluster returns a clientset for the cluster, if present.
 	GetClientsetForCluster(clusterName string) (kubeclientset.Interface, error)
 
+	// GetUnreadyClusters returns a list of all clusters that are not ready yet.
+	GetUnreadyClusters() ([]*federation_api.Cluster, error)
+
 	// GetReadyClusers returns all clusters for which the sub-informers are run.
 	GetReadyClusters() ([]*federation_api.Cluster, error)
 
@@ -260,6 +263,9 @@ type federatedInformerImpl struct {
 	clientFactory func(*federation_api.Cluster) (kubeclientset.Interface, error)
 }
 
+// *federatedInformerImpl implements FederatedInformer interface.
+var _ FederatedInformer = &federatedInformerImpl{}
+
 type federatedStoreImpl struct {
 	federatedInformer *federatedInformerImpl
 }
@@ -313,6 +319,24 @@ func (f *federatedInformerImpl) getClientsetForClusterUnlocked(clusterName strin
 	return nil, fmt.Errorf("cluster %q not found", clusterName)
 }
 
+func (f *federatedInformerImpl) GetUnreadyClusters() ([]*federation_api.Cluster, error) {
+	f.Lock()
+	defer f.Unlock()
+
+	items := f.clusterInformer.store.List()
+	result := make([]*federation_api.Cluster, 0, len(items))
+	for _, item := range items {
+		if cluster, ok := item.(*federation_api.Cluster); ok {
+			if !isClusterReady(cluster) {
+				result = append(result, cluster)
+			}
+		} else {
+			return nil, fmt.Errorf("wrong data in FederatedInformerImpl cluster store: %v", item)
+		}
+	}
+	return result, nil
+}
+
 // GetReadyClusers returns all clusters for which the sub-informers are run.
 func (f *federatedInformerImpl) GetReadyClusters() ([]*federation_api.Cluster, error) {
 	f.Lock()

federation/pkg/federation-controller/util/federated_updater_test.go

@@ -34,7 +34,10 @@ import (
 type fakeFederationView struct {
 }
 
-func (f fakeFederationView) GetClientsetForCluster(clusterName string) (kubeclientset.Interface, error) {
+// Verify that fakeFederationView implements FederationView interface
+var _ FederationView = &fakeFederationView{}
+
+func (f *fakeFederationView) GetClientsetForCluster(clusterName string) (kubeclientset.Interface, error) {
 	return &fake_kubeclientset.Clientset{}, nil
 }
 
@@ -42,6 +45,10 @@ func (f *fakeFederationView) GetReadyClusters() ([]*federation_api.Cluster, erro
 	return []*federation_api.Cluster{}, nil
 }
 
+func (f *fakeFederationView) GetUnreadyClusters() ([]*federation_api.Cluster, error) {
+	return []*federation_api.Cluster{}, nil
+}
+
 func (f *fakeFederationView) GetReadyCluster(name string) (*federation_api.Cluster, bool, error) {
 	return nil, false, nil
 }

federation/pkg/federation-controller/util/test/test_helper.go

@@ -163,30 +163,35 @@ func RegisterFakeCopyOnCreate(resource string, client *core.Fake, watcher *Watch
 	objChan := make(chan runtime.Object, 100)
 	client.AddReactor("create", resource, func(action core.Action) (bool, runtime.Object, error) {
 		createAction := action.(core.CreateAction)
-		obj := createAction.GetObject()
+		originalObj := createAction.GetObject()
+		// Create a copy of the object here to prevent data races while reading the object in go routine.
+		obj := copy(originalObj)
 		go func() {
+			glog.V(4).Infof("Object created. Writing to channel: %v", obj)
 			watcher.Add(obj)
-			objChan <- copy(obj)
+			objChan <- obj
 		}()
-		return true, obj, nil
+		return true, originalObj, nil
 	})
 	return objChan
 }
 
-// RegisterFakeCopyOnCreate registers a reactor in the given fake client that passes
+// RegisterFakeCopyOnUpdate registers a reactor in the given fake client that passes
 // all updated objects to the given watcher and also copies them to a channel for
 // in-test inspection.
 func RegisterFakeCopyOnUpdate(resource string, client *core.Fake, watcher *WatcherDispatcher) chan runtime.Object {
 	objChan := make(chan runtime.Object, 100)
 	client.AddReactor("update", resource, func(action core.Action) (bool, runtime.Object, error) {
 		updateAction := action.(core.UpdateAction)
-		obj := updateAction.GetObject()
+		originalObj := updateAction.GetObject()
+		// Create a copy of the object here to prevent data races while reading the object in go routine.
+		obj := copy(originalObj)
 		go func() {
 			glog.V(4).Infof("Object updated. Writing to channel: %v", obj)
 			watcher.Modify(obj)
-			objChan <- copy(obj)
+			objChan <- obj
 		}()
-		return true, obj, nil
+		return true, originalObj, nil
 	})
 	return objChan
 }