Implemented removal of Deployments, Daemons & HorizontalPodAutoscalers when Namespace is removed.

Implemented removal of Deployments, Daemons & HorizontalPodAutoscalers when Namespace is removed. Added unittest. Fixes #12735.

cmd/kube-controller-manager/app/controllermanager.go

@@ -242,9 +242,16 @@ func (s *CMServer) Run(_ []string) error {
 	resourceQuotaController := resourcequotacontroller.NewResourceQuotaController(kubeClient)
 	resourceQuotaController.Run(s.ResourceQuotaSyncPeriod)
 
-	namespaceController := namespacecontroller.NewNamespaceController(kubeClient, s.NamespaceSyncPeriod)
+	// An OR of all flags to enable/disable experimental features
+	experimentalMode := s.EnableHorizontalPodAutoscaler
+	namespaceController := namespacecontroller.NewNamespaceController(kubeClient, experimentalMode, s.NamespaceSyncPeriod)
 	namespaceController.Run()
 
+	if s.EnableHorizontalPodAutoscaler {
+		horizontalPodAutoscalerController := autoscalercontroller.New(kubeClient, metrics.NewHeapsterMetricsClient(kubeClient))
+		horizontalPodAutoscalerController.Run(s.HorizontalPodAutoscalerSyncPeriod)
+	}
+
 	pvclaimBinder := volumeclaimbinder.NewPersistentVolumeClaimBinder(kubeClient, s.PVClaimBinderSyncPeriod)
 	pvclaimBinder.Run()
 	pvRecycler, err := volumeclaimbinder.NewPersistentVolumeRecycler(kubeClient, s.PVClaimBinderSyncPeriod, ProbeRecyclableVolumePlugins(s.VolumeConfigFlags))
@@ -287,15 +294,5 @@ func (s *CMServer) Run(_ []string) error {
 		serviceaccount.DefaultServiceAccountsControllerOptions(),
 	).Run()
 
-	if s.EnableHorizontalPodAutoscaler {
-		expClient, err := client.NewExperimental(kubeconfig)
-		if err != nil {
-			glog.Fatalf("Invalid API configuration: %v", err)
-		}
-		horizontalPodAutoscalerController := autoscalercontroller.New(kubeClient, expClient,
-			metrics.NewHeapsterMetricsClient(kubeClient))
-		horizontalPodAutoscalerController.Run(s.HorizontalPodAutoscalerSyncPeriod)
-	}
-
 	select {}
 }

contrib/mesos/pkg/controllermanager/controllermanager.go

@@ -144,7 +144,7 @@ func (s *CMServer) Run(_ []string) error {
 	resourceQuotaController := resourcequotacontroller.NewResourceQuotaController(kubeClient)
 	resourceQuotaController.Run(s.ResourceQuotaSyncPeriod)
 
-	namespaceController := namespacecontroller.NewNamespaceController(kubeClient, s.NamespaceSyncPeriod)
+	namespaceController := namespacecontroller.NewNamespaceController(kubeClient, false, s.NamespaceSyncPeriod)
 	namespaceController.Run()
 
 	pvclaimBinder := volumeclaimbinder.NewPersistentVolumeClaimBinder(kubeClient, s.PVClaimBinderSyncPeriod)

pkg/client/unversioned/testclient/fake_deployments.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testclient
+
+import (
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/expapi"
+	"k8s.io/kubernetes/pkg/fields"
+	"k8s.io/kubernetes/pkg/labels"
+	"k8s.io/kubernetes/pkg/watch"
+)
+
+// FakeDeployments implements DeploymentsInterface. Meant to be embedded into a struct to get a default
+// implementation. This makes faking out just the methods you want to test easier.
+type FakeDeployments struct {
+	Fake      *FakeExperimental
+	Namespace string
+}
+
+func (c *FakeDeployments) Get(name string) (*expapi.Deployment, error) {
+	obj, err := c.Fake.Invokes(NewGetAction("deployments", c.Namespace, name), &expapi.Deployment{})
+	if obj == nil {
+		return nil, err
+	}
+
+	return obj.(*expapi.Deployment), err
+}
+
+func (c *FakeDeployments) List(label labels.Selector, field fields.Selector) (*expapi.DeploymentList, error) {
+	obj, err := c.Fake.Invokes(NewListAction("deployments", c.Namespace, label, field), &expapi.DeploymentList{})
+	if obj == nil {
+		return nil, err
+	}
+	list := &expapi.DeploymentList{}
+	for _, deployment := range obj.(*expapi.DeploymentList).Items {
+		if label.Matches(labels.Set(deployment.Labels)) {
+			list.Items = append(list.Items, deployment)
+		}
+	}
+	return list, err
+}
+
+func (c *FakeDeployments) Create(deployment *expapi.Deployment) (*expapi.Deployment, error) {
+	obj, err := c.Fake.Invokes(NewCreateAction("deployments", c.Namespace, deployment), deployment)
+	if obj == nil {
+		return nil, err
+	}
+
+	return obj.(*expapi.Deployment), err
+}
+
+func (c *FakeDeployments) Update(deployment *expapi.Deployment) (*expapi.Deployment, error) {
+	obj, err := c.Fake.Invokes(NewUpdateAction("deployments", c.Namespace, deployment), deployment)
+	if obj == nil {
+		return nil, err
+	}
+
+	return obj.(*expapi.Deployment), err
+}
+
+func (c *FakeDeployments) Delete(name string, options *api.DeleteOptions) error {
+	_, err := c.Fake.Invokes(NewDeleteAction("deployments", c.Namespace, name), &expapi.Deployment{})
+	return err
+}
+
+func (c *FakeDeployments) Watch(label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) {
+	return c.Fake.InvokesWatch(NewWatchAction("deployments", c.Namespace, label, field, resourceVersion))
+}

pkg/client/unversioned/testclient/fake_horizontal_pod_autoscalers.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testclient
+
+import (
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/expapi"
+	"k8s.io/kubernetes/pkg/fields"
+	"k8s.io/kubernetes/pkg/labels"
+	"k8s.io/kubernetes/pkg/watch"
+)
+
+// FakeHorizontalPodAutoscalers implements HorizontalPodAutoscalerInterface. Meant to be embedded into a struct to get a default
+// implementation. This makes faking out just the methods you want to test easier.
+type FakeHorizontalPodAutoscalers struct {
+	Fake      *FakeExperimental
+	Namespace string
+}
+
+func (c *FakeHorizontalPodAutoscalers) Get(name string) (*expapi.HorizontalPodAutoscaler, error) {
+	obj, err := c.Fake.Invokes(NewGetAction("horizontalpodautoscalers", c.Namespace, name), &expapi.HorizontalPodAutoscaler{})
+	if obj == nil {
+		return nil, err
+	}
+
+	return obj.(*expapi.HorizontalPodAutoscaler), err
+}
+
+func (c *FakeHorizontalPodAutoscalers) List(label labels.Selector, field fields.Selector) (*expapi.HorizontalPodAutoscalerList, error) {
+	obj, err := c.Fake.Invokes(NewListAction("horizontalpodautoscalers", c.Namespace, label, field), &expapi.HorizontalPodAutoscalerList{})
+	if obj == nil {
+		return nil, err
+	}
+	list := &expapi.HorizontalPodAutoscalerList{}
+	for _, a := range obj.(*expapi.HorizontalPodAutoscalerList).Items {
+		if label.Matches(labels.Set(a.Labels)) {
+			list.Items = append(list.Items, a)
+		}
+	}
+	return list, err
+}
+
+func (c *FakeHorizontalPodAutoscalers) Create(a *expapi.HorizontalPodAutoscaler) (*expapi.HorizontalPodAutoscaler, error) {
+	obj, err := c.Fake.Invokes(NewCreateAction("horizontalpodautoscalers", c.Namespace, a), a)
+	if obj == nil {
+		return nil, err
+	}
+
+	return obj.(*expapi.HorizontalPodAutoscaler), err
+}
+
+func (c *FakeHorizontalPodAutoscalers) Update(a *expapi.HorizontalPodAutoscaler) (*expapi.HorizontalPodAutoscaler, error) {
+	obj, err := c.Fake.Invokes(NewUpdateAction("horizontalpodautoscalers", c.Namespace, a), a)
+	if obj == nil {
+		return nil, err
+	}
+
+	return obj.(*expapi.HorizontalPodAutoscaler), err
+}
+
+func (c *FakeHorizontalPodAutoscalers) Delete(name string, options *api.DeleteOptions) error {
+	_, err := c.Fake.Invokes(NewDeleteAction("horizontalpodautoscalers", c.Namespace, name), &expapi.HorizontalPodAutoscaler{})
+	return err
+}
+
+func (c *FakeHorizontalPodAutoscalers) Watch(label labels.Selector, field fields.Selector, resourceVersion string) (watch.Interface, error) {
+	return c.Fake.InvokesWatch(NewWatchAction("horizontalpodautoscalers", c.Namespace, label, field, resourceVersion))
+}

pkg/client/unversioned/testclient/fake_scales.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testclient
+
+import (
+	"k8s.io/kubernetes/pkg/expapi"
+)
+
+// FakeScales implements ScaleInterface. Meant to be embedded into a struct to get a default
+// implementation. This makes faking out just the methods you want to test easier.
+type FakeScales struct {
+	Fake      *FakeExperimental
+	Namespace string
+}
+
+func (c *FakeScales) Get(kind string, name string) (result *expapi.Scale, err error) {
+	action := GetActionImpl{}
+	action.Verb = "get"
+	action.Namespace = c.Namespace
+	action.Resource = kind
+	action.Subresource = "scale"
+	action.Name = name
+	obj, err := c.Fake.Invokes(action, &expapi.Scale{})
+	result = obj.(*expapi.Scale)
+	return
+}
+
+func (c *FakeScales) Update(kind string, scale *expapi.Scale) (result *expapi.Scale, err error) {
+	action := UpdateActionImpl{}
+	action.Verb = "update"
+	action.Namespace = c.Namespace
+	action.Resource = kind
+	action.Subresource = "scale"
+	action.Object = scale
+	obj, err := c.Fake.Invokes(action, scale)
+	result = obj.(*expapi.Scale)
+	return
+}

pkg/client/unversioned/testclient/testclient.go

@@ -251,13 +251,13 @@ func (c *FakeExperimental) Daemons(namespace string) client.DaemonInterface {
 }
 
 func (c *FakeExperimental) HorizontalPodAutoscalers(namespace string) client.HorizontalPodAutoscalerInterface {
-	panic("unimplemented")
+	return &FakeHorizontalPodAutoscalers{Fake: c, Namespace: namespace}
-}
-
-func (c *FakeExperimental) Scales(namespace string) client.ScaleInterface {
-	panic("unimplemented")
 }
 
 func (c *FakeExperimental) Deployments(namespace string) client.DeploymentInterface {
-	panic("unimplemented")
+	return &FakeDeployments{Fake: c, Namespace: namespace}
+}
+
+func (c *FakeExperimental) Scales(namespace string) client.ScaleInterface {
+	return &FakeScales{Fake: c, Namespace: namespace}
 }

pkg/controller/autoscaler/horizontalpodautoscaler_controller.go

@@ -42,17 +42,15 @@ const (
 
 type HorizontalPodAutoscalerController struct {
 	client        client.Interface
-	expClient     client.ExperimentalInterface
 	metricsClient metrics.MetricsClient
 }
 
 var downscaleForbiddenWindow, _ = time.ParseDuration("20m")
 var upscaleForbiddenWindow, _ = time.ParseDuration("3m")
 
-func New(client client.Interface, expClient client.ExperimentalInterface, metricsClient metrics.MetricsClient) *HorizontalPodAutoscalerController {
+func New(client client.Interface, metricsClient metrics.MetricsClient) *HorizontalPodAutoscalerController {
 	return &HorizontalPodAutoscalerController{
 		client:        client,
-		expClient:     expClient,
 		metricsClient: metricsClient,
 	}
 }
@@ -67,14 +65,14 @@ func (a *HorizontalPodAutoscalerController) Run(syncPeriod time.Duration) {
 
 func (a *HorizontalPodAutoscalerController) reconcileAutoscalers() error {
 	ns := api.NamespaceAll
-	list, err := a.expClient.HorizontalPodAutoscalers(ns).List(labels.Everything(), fields.Everything())
+	list, err := a.client.Experimental().HorizontalPodAutoscalers(ns).List(labels.Everything(), fields.Everything())
 	if err != nil {
 		return fmt.Errorf("error listing nodes: %v", err)
 	}
 	for _, hpa := range list.Items {
 		reference := fmt.Sprintf("%s/%s/%s", hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Namespace, hpa.Spec.ScaleRef.Name)
 
-		scale, err := a.expClient.Scales(hpa.Spec.ScaleRef.Namespace).Get(hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Name)
+		scale, err := a.client.Experimental().Scales(hpa.Spec.ScaleRef.Namespace).Get(hpa.Spec.ScaleRef.Kind, hpa.Spec.ScaleRef.Name)
 		if err != nil {
 			glog.Warningf("Failed to query scale subresource for %s: %v", reference, err)
 			continue
@@ -127,7 +125,7 @@ func (a *HorizontalPodAutoscalerController) reconcileAutoscalers() error {
 
 		if rescale {
 			scale.Spec.Replicas = desiredReplicas
-			_, err = a.expClient.Scales(hpa.Namespace).Update(hpa.Spec.ScaleRef.Kind, scale)
+			_, err = a.client.Experimental().Scales(hpa.Namespace).Update(hpa.Spec.ScaleRef.Kind, scale)
 			if err != nil {
 				glog.Warningf("Failed to rescale %s: %v", reference, err)
 				continue
@@ -147,7 +145,7 @@ func (a *HorizontalPodAutoscalerController) reconcileAutoscalers() error {
 			hpa.Status.LastScaleTimestamp = &now
 		}
 
-		_, err = a.expClient.HorizontalPodAutoscalers(hpa.Namespace).Update(&hpa)
+		_, err = a.client.Experimental().HorizontalPodAutoscalers(hpa.Namespace).Update(&hpa)
 		if err != nil {
 			glog.Warningf("Failed to update HorizontalPodAutoscaler %s: %v", hpa.Name, err)
 			continue

pkg/controller/autoscaler/horizontalpodautoscaler_controller_test.go

@@ -177,14 +177,12 @@ func TestSyncEndpointsItemsPreserveNoSelector(t *testing.T) {
 
 	defer testServer.Close()
 	kubeClient := client.NewOrDie(&client.Config{Host: testServer.URL, Version: testapi.Experimental.Version()})
-	expClient := client.NewExperimentalOrDie(&client.Config{Host: testServer.URL, Version: testapi.Experimental.Version()})
-
 	fakeRC := fakeResourceConsumptionClient{metrics: map[api.ResourceName]expapi.ResourceConsumption{
 		api.ResourceCPU: {Resource: api.ResourceCPU, Quantity: resource.MustParse("650m")},
 	}}
 	fake := fakeMetricsClient{consumption: &fakeRC}
 
-	hpaController := New(kubeClient, expClient, &fake)
+	hpaController := New(kubeClient, &fake)
 
 	err := hpaController.reconcileAutoscalers()
 	if err != nil {
@@ -193,7 +191,7 @@ func TestSyncEndpointsItemsPreserveNoSelector(t *testing.T) {
 	for _, h := range handlers {
 		h.ValidateRequestCount(t, 1)
 	}
-	obj, err := expClient.Codec.Decode([]byte(handlers[updateHpaHandler].RequestBody))
+	obj, err := kubeClient.Codec.Decode([]byte(handlers[updateHpaHandler].RequestBody))
 	if err != nil {
 		t.Fatal("Failed to decode: %v %v", err)
 	}

pkg/controller/namespace/namespace_controller.go

@@ -41,7 +41,7 @@ type NamespaceController struct {
 }
 
 // NewNamespaceController creates a new NamespaceController
-func NewNamespaceController(kubeClient client.Interface, resyncPeriod time.Duration) *NamespaceController {
+func NewNamespaceController(kubeClient client.Interface, experimentalMode bool, resyncPeriod time.Duration) *NamespaceController {
 	var controller *framework.Controller
 	_, controller = framework.NewInformer(
 		&cache.ListWatch{
@@ -57,7 +57,7 @@ func NewNamespaceController(kubeClient client.Interface, resyncPeriod time.Durat
 		framework.ResourceEventHandlerFuncs{
 			AddFunc: func(obj interface{}) {
 				namespace := obj.(*api.Namespace)
-				if err := syncNamespace(kubeClient, *namespace); err != nil {
+				if err := syncNamespace(kubeClient, experimentalMode, *namespace); err != nil {
 					if estimate, ok := err.(*contentRemainingError); ok {
 						go func() {
 							// Estimate is the aggregate total of TerminationGracePeriodSeconds, which defaults to 30s
@@ -79,7 +79,7 @@ func NewNamespaceController(kubeClient client.Interface, resyncPeriod time.Durat
 			},
 			UpdateFunc: func(oldObj, newObj interface{}) {
 				namespace := newObj.(*api.Namespace)
-				if err := syncNamespace(kubeClient, *namespace); err != nil {
+				if err := syncNamespace(kubeClient, experimentalMode, *namespace); err != nil {
 					if estimate, ok := err.(*contentRemainingError); ok {
 						go func() {
 							t := estimate.Estimate/2 + 1
@@ -152,7 +152,7 @@ func (e *contentRemainingError) Error() string {
 // deleteAllContent will delete all content known to the system in a namespace. It returns an estimate
 // of the time remaining before the remaining resources are deleted. If estimate > 0 not all resources
 // are guaranteed to be gone.
-func deleteAllContent(kubeClient client.Interface, namespace string, before util.Time) (estimate int64, err error) {
+func deleteAllContent(kubeClient client.Interface, experimentalMode bool, namespace string, before util.Time) (estimate int64, err error) {
 	err = deleteServiceAccounts(kubeClient, namespace)
 	if err != nil {
 		return estimate, err
@@ -189,12 +189,26 @@ func deleteAllContent(kubeClient client.Interface, namespace string, before util
 	if err != nil {
 		return estimate, err
 	}
-
+	// If experimental mode, delete all experimental resources for the namespace.
+	if experimentalMode {
+		err = deleteHorizontalPodAutoscalers(kubeClient.Experimental(), namespace)
+		if err != nil {
+			return estimate, err
+		}
+		err = deleteDaemons(kubeClient.Experimental(), namespace)
+		if err != nil {
+			return estimate, err
+		}
+		err = deleteDeployments(kubeClient.Experimental(), namespace)
+		if err != nil {
+			return estimate, err
+		}
+	}
 	return estimate, nil
 }
 
 // syncNamespace makes namespace life-cycle decisions
-func syncNamespace(kubeClient client.Interface, namespace api.Namespace) (err error) {
+func syncNamespace(kubeClient client.Interface, experimentalMode bool, namespace api.Namespace) (err error) {
 	if namespace.DeletionTimestamp == nil {
 		return nil
 	}
@@ -224,7 +238,7 @@ func syncNamespace(kubeClient client.Interface, namespace api.Namespace) (err er
 	}
 
 	// there may still be content for us to remove
-	estimate, err := deleteAllContent(kubeClient, namespace.Name, *namespace.DeletionTimestamp)
+	estimate, err := deleteAllContent(kubeClient, experimentalMode, namespace.Name, *namespace.DeletionTimestamp)
 	if err != nil {
 		return err
 	}
@@ -389,3 +403,45 @@ func deletePersistentVolumeClaims(kubeClient client.Interface, ns string) error
 	}
 	return nil
 }
+
+func deleteHorizontalPodAutoscalers(expClient client.ExperimentalInterface, ns string) error {
+	items, err := expClient.HorizontalPodAutoscalers(ns).List(labels.Everything(), fields.Everything())
+	if err != nil {
+		return err
+	}
+	for i := range items.Items {
+		err := expClient.HorizontalPodAutoscalers(ns).Delete(items.Items[i].Name, nil)
+		if err != nil && !errors.IsNotFound(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+func deleteDaemons(expClient client.ExperimentalInterface, ns string) error {
+	items, err := expClient.Daemons(ns).List(labels.Everything())
+	if err != nil {
+		return err
+	}
+	for i := range items.Items {
+		err := expClient.Daemons(ns).Delete(items.Items[i].Name)
+		if err != nil && !errors.IsNotFound(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+func deleteDeployments(expClient client.ExperimentalInterface, ns string) error {
+	items, err := expClient.Deployments(ns).List(labels.Everything(), fields.Everything())
+	if err != nil {
+		return err
+	}
+	for i := range items.Items {
+		err := expClient.Deployments(ns).Delete(items.Items[i].Name, nil)
+		if err != nil && !errors.IsNotFound(err) {
+			return err
+		}
+	}
+	return nil
+}

pkg/controller/namespace/namespace_controller_test.go

@@ -69,7 +69,7 @@ func TestFinalize(t *testing.T) {
 	}
 }
 
-func TestSyncNamespaceThatIsTerminating(t *testing.T) {
+func testSyncNamespaceThatIsTerminating(t *testing.T, experimentalMode bool) {
 	mockClient := &testclient.Fake{}
 	now := util.Now()
 	testNamespace := api.Namespace{
@@ -85,7 +85,7 @@ func TestSyncNamespaceThatIsTerminating(t *testing.T) {
 			Phase: api.NamespaceTerminating,
 		},
 	}
-	err := syncNamespace(mockClient, testNamespace)
+	err := syncNamespace(mockClient, experimentalMode, testNamespace)
 	if err != nil {
 		t.Errorf("Unexpected error when synching namespace %v", err)
 	}
@@ -98,9 +98,20 @@ func TestSyncNamespaceThatIsTerminating(t *testing.T) {
 		strings.Join([]string{"list", "secrets", ""}, "-"),
 		strings.Join([]string{"list", "limitranges", ""}, "-"),
 		strings.Join([]string{"list", "events", ""}, "-"),
+		strings.Join([]string{"list", "serviceaccounts", ""}, "-"),
+		strings.Join([]string{"list", "persistentvolumeclaims", ""}, "-"),
 		strings.Join([]string{"create", "namespaces", "finalize"}, "-"),
 		strings.Join([]string{"delete", "namespaces", ""}, "-"),
 	)
+
+	if experimentalMode {
+		expectedActionSet.Insert(
+			strings.Join([]string{"list", "horizontalpodautoscalers", ""}, "-"),
+			strings.Join([]string{"list", "daemons", ""}, "-"),
+			strings.Join([]string{"list", "deployments", ""}, "-"),
+		)
+	}
+
 	actionSet := util.NewStringSet()
 	for _, action := range mockClient.Actions() {
 		actionSet.Insert(strings.Join([]string{action.GetVerb(), action.GetResource(), action.GetSubresource()}, "-"))
@@ -108,6 +119,17 @@ func TestSyncNamespaceThatIsTerminating(t *testing.T) {
 	if !actionSet.HasAll(expectedActionSet.List()...) {
 		t.Errorf("Expected actions: %v, but got: %v", expectedActionSet, actionSet)
 	}
+	if !expectedActionSet.HasAll(actionSet.List()...) {
+		t.Errorf("Expected actions: %v, but got: %v", expectedActionSet, actionSet)
+	}
+}
+
+func TestSyncNamespaceThatIsTerminatingNonExperimental(t *testing.T) {
+	testSyncNamespaceThatIsTerminating(t, false)
+}
+
+func TestSyncNamespaceThatIsTerminatingExperimental(t *testing.T) {
+	testSyncNamespaceThatIsTerminating(t, true)
 }
 
 func TestSyncNamespaceThatIsActive(t *testing.T) {
@@ -124,7 +146,7 @@ func TestSyncNamespaceThatIsActive(t *testing.T) {
 			Phase: api.NamespaceActive,
 		},
 	}
-	err := syncNamespace(mockClient, testNamespace)
+	err := syncNamespace(mockClient, false, testNamespace)
 	if err != nil {
 		t.Errorf("Unexpected error when synching namespace %v", err)
 	}
@@ -135,7 +157,7 @@ func TestSyncNamespaceThatIsActive(t *testing.T) {
 
 func TestRunStop(t *testing.T) {
 	mockClient := &testclient.Fake{}
-	nsController := NewNamespaceController(mockClient, 1*time.Second)
+	nsController := NewNamespaceController(mockClient, false, 1*time.Second)
 
 	if nsController.StopEverything != nil {
 		t.Errorf("Non-running manager should not have a stop channel.  Got %v", nsController.StopEverything)