Merge pull request #27128 from markturansky/disable_provisioning

Automatic merge from submit-queue

Allow disabling of dynamic provisioning

Allow administrators to opt-out of dynamic provisioning.  Provisioning is still on by default, which is the current behavior.

Per a conversation with @jsafrane, a boolean toggle was added and plumbed through into the controller.  Deliberate disabling will simply return nil from `provisionClaim` whereas a misconfigured provisioner will continue on and generate error events for the PVC.

@kubernetes/rh-storage @saad-ali @thockin  @abhgupta

cmd/kube-controller-manager/app/controllermanager.go

@@ -399,6 +399,7 @@ func StartControllers(s *options.CMServer, kubeClient *client.Client, kubeconfig
 		cloud,
 		s.ClusterName,
 		nil, nil, nil,
+		s.VolumeConfiguration.EnableDynamicProvisioning,
 	)
 	volumeController.Run()
 	time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter))

cmd/kube-controller-manager/app/options/options.go

@@ -75,6 +75,7 @@ func NewCMServer() *CMServer {
 			TerminatedPodGCThreshold:          12500,
 			VolumeConfiguration: componentconfig.VolumeConfiguration{
 				EnableHostPathProvisioning: false,
+				EnableDynamicProvisioning:  true,
 				PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
 					MaximumRetry:             3,
 					MinimumTimeoutNFS:        300,
@@ -125,6 +126,7 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet) {
 	fs.Int32Var(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutHostPath, "pv-recycler-minimum-timeout-hostpath", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutHostPath, "The minimum ActiveDeadlineSeconds to use for a HostPath Recycler pod.  This is for development and testing only and will not work in a multi-node cluster.")
 	fs.Int32Var(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutHostPath, "pv-recycler-timeout-increment-hostpath", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutHostPath, "the increment of time added per Gi to ActiveDeadlineSeconds for a HostPath scrubber pod.  This is for development and testing only and will not work in a multi-node cluster.")
 	fs.BoolVar(&s.VolumeConfiguration.EnableHostPathProvisioning, "enable-hostpath-provisioner", s.VolumeConfiguration.EnableHostPathProvisioning, "Enable HostPath PV provisioning when running without a cloud provider. This allows testing and development of provisioning features.  HostPath provisioning is not supported in any way, won't work in a multi-node cluster, and should not be used for anything other than testing or development.")
+	fs.BoolVar(&s.VolumeConfiguration.EnableDynamicProvisioning, "enable-dynamic-provisioning", s.VolumeConfiguration.EnableDynamicProvisioning, "Enable dynamic provisioning for environments that support it.")
 	fs.StringVar(&s.VolumeConfiguration.FlexVolumePluginDir, "flex-volume-plugin-dir", s.VolumeConfiguration.FlexVolumePluginDir, "Full path of the directory in which the flex volume plugin should search for additional third party volume plugins.")
 	fs.Int32Var(&s.TerminatedPodGCThreshold, "terminated-pod-gc-threshold", s.TerminatedPodGCThreshold, "Number of terminated pods that can exist before the terminated pod garbage collector starts deleting terminated pods. If <= 0, the terminated pod garbage collector is disabled.")
 	fs.DurationVar(&s.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", s.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.")

contrib/mesos/pkg/controllermanager/controllermanager.go

@@ -287,6 +287,7 @@ func (s *CMServer) Run(_ []string) error {
 		nil,
 		nil,
 		nil,
+		s.VolumeConfiguration.EnableDynamicProvisioning,
 	)
 	volumeController.Run()
 

hack/verify-flags/known-flags.txt

@@ -119,6 +119,7 @@ enable-controller-attach-detach
 enable-custom-metrics
 enable-debugging-handlers
 enable-garbage-collector
+enable-dynamic-provisioning
 enable-hostpath-provisioner
 enable-server
 enable-swagger-ui

pkg/apis/componentconfig/deep_copy_generated.go

@@ -361,6 +361,7 @@ func DeepCopy_componentconfig_PortRangeVar(in PortRangeVar, out *PortRangeVar, c
 
 func DeepCopy_componentconfig_VolumeConfiguration(in VolumeConfiguration, out *VolumeConfiguration, c *conversion.Cloner) error {
 	out.EnableHostPathProvisioning = in.EnableHostPathProvisioning
+	out.EnableDynamicProvisioning = in.EnableDynamicProvisioning
 	if err := DeepCopy_componentconfig_PersistentVolumeRecyclerConfiguration(in.PersistentVolumeRecyclerConfiguration, &out.PersistentVolumeRecyclerConfiguration, c); err != nil {
 		return err
 	}

pkg/apis/componentconfig/types.generated.go

@@ -8914,14 +8914,14 @@ func (x *VolumeConfiguration) CodecEncodeSelf(e *codec1978.Encoder) {
 		} else {
 			yysep2 := !z.EncBinary()
 			yy2arr2 := z.EncBasicHandle().StructToArray
-			var yyq2 [3]bool
+			var yyq2 [4]bool
 			_, _, _ = yysep2, yyq2, yy2arr2
 			const yyr2 bool = false
 			var yynn2 int
 			if yyr2 || yy2arr2 {
-				r.EncodeArrayStart(3)
+				r.EncodeArrayStart(4)
 			} else {
-				yynn2 = 3
+				yynn2 = 4
 				for _, b := range yyq2 {
 					if b {
 						yynn2++
@@ -8951,19 +8951,38 @@ func (x *VolumeConfiguration) CodecEncodeSelf(e *codec1978.Encoder) {
 			}
 			if yyr2 || yy2arr2 {
 				z.EncSendContainerState(codecSelfer_containerArrayElem1234)
-				yy7 := &x.PersistentVolumeRecyclerConfiguration
+				yym7 := z.EncBinary()
-				yy7.CodecEncodeSelf(e)
+				_ = yym7
+				if false {
+				} else {
+					r.EncodeBool(bool(x.EnableDynamicProvisioning))
+				}
+			} else {
+				z.EncSendContainerState(codecSelfer_containerMapKey1234)
+				r.EncodeString(codecSelferC_UTF81234, string("enableDynamicProvisioning"))
+				z.EncSendContainerState(codecSelfer_containerMapValue1234)
+				yym8 := z.EncBinary()
+				_ = yym8
+				if false {
+				} else {
+					r.EncodeBool(bool(x.EnableDynamicProvisioning))
+				}
+			}
+			if yyr2 || yy2arr2 {
+				z.EncSendContainerState(codecSelfer_containerArrayElem1234)
+				yy10 := &x.PersistentVolumeRecyclerConfiguration
+				yy10.CodecEncodeSelf(e)
 			} else {
 				z.EncSendContainerState(codecSelfer_containerMapKey1234)
 				r.EncodeString(codecSelferC_UTF81234, string("persitentVolumeRecyclerConfiguration"))
 				z.EncSendContainerState(codecSelfer_containerMapValue1234)
-				yy9 := &x.PersistentVolumeRecyclerConfiguration
+				yy12 := &x.PersistentVolumeRecyclerConfiguration
-				yy9.CodecEncodeSelf(e)
+				yy12.CodecEncodeSelf(e)
 			}
 			if yyr2 || yy2arr2 {
 				z.EncSendContainerState(codecSelfer_containerArrayElem1234)
-				yym12 := z.EncBinary()
+				yym15 := z.EncBinary()
-				_ = yym12
+				_ = yym15
 				if false {
 				} else {
 					r.EncodeString(codecSelferC_UTF81234, string(x.FlexVolumePluginDir))
@@ -8972,8 +8991,8 @@ func (x *VolumeConfiguration) CodecEncodeSelf(e *codec1978.Encoder) {
 				z.EncSendContainerState(codecSelfer_containerMapKey1234)
 				r.EncodeString(codecSelferC_UTF81234, string("flexVolumePluginDir"))
 				z.EncSendContainerState(codecSelfer_containerMapValue1234)
-				yym13 := z.EncBinary()
+				yym16 := z.EncBinary()
-				_ = yym13
+				_ = yym16
 				if false {
 				} else {
 					r.EncodeString(codecSelferC_UTF81234, string(x.FlexVolumePluginDir))
@@ -9046,12 +9065,18 @@ func (x *VolumeConfiguration) codecDecodeSelfFromMap(l int, d *codec1978.Decoder
 			} else {
 				x.EnableHostPathProvisioning = bool(r.DecodeBool())
 			}
+		case "enableDynamicProvisioning":
+			if r.TryDecodeAsNil() {
+				x.EnableDynamicProvisioning = false
+			} else {
+				x.EnableDynamicProvisioning = bool(r.DecodeBool())
+			}
 		case "persitentVolumeRecyclerConfiguration":
 			if r.TryDecodeAsNil() {
 				x.PersistentVolumeRecyclerConfiguration = PersistentVolumeRecyclerConfiguration{}
 			} else {
-				yyv5 := &x.PersistentVolumeRecyclerConfiguration
+				yyv6 := &x.PersistentVolumeRecyclerConfiguration
-				yyv5.CodecDecodeSelf(d)
+				yyv6.CodecDecodeSelf(d)
 			}
 		case "flexVolumePluginDir":
 			if r.TryDecodeAsNil() {
@@ -9070,16 +9095,16 @@ func (x *VolumeConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Decod
 	var h codecSelfer1234
 	z, r := codec1978.GenHelperDecoder(d)
 	_, _, _ = h, z, r
-	var yyj7 int
+	var yyj8 int
-	var yyb7 bool
+	var yyb8 bool
-	var yyhl7 bool = l >= 0
+	var yyhl8 bool = l >= 0
-	yyj7++
+	yyj8++
-	if yyhl7 {
+	if yyhl8 {
-		yyb7 = yyj7 > l
+		yyb8 = yyj8 > l
 	} else {
-		yyb7 = r.CheckBreak()
+		yyb8 = r.CheckBreak()
 	}
-	if yyb7 {
+	if yyb8 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -9089,13 +9114,29 @@ func (x *VolumeConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Decod
 	} else {
 		x.EnableHostPathProvisioning = bool(r.DecodeBool())
 	}
-	yyj7++
+	yyj8++
-	if yyhl7 {
+	if yyhl8 {
-		yyb7 = yyj7 > l
+		yyb8 = yyj8 > l
 	} else {
-		yyb7 = r.CheckBreak()
+		yyb8 = r.CheckBreak()
 	}
-	if yyb7 {
+	if yyb8 {
+		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
+		return
+	}
+	z.DecSendContainerState(codecSelfer_containerArrayElem1234)
+	if r.TryDecodeAsNil() {
+		x.EnableDynamicProvisioning = false
+	} else {
+		x.EnableDynamicProvisioning = bool(r.DecodeBool())
+	}
+	yyj8++
+	if yyhl8 {
+		yyb8 = yyj8 > l
+	} else {
+		yyb8 = r.CheckBreak()
+	}
+	if yyb8 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -9103,16 +9144,16 @@ func (x *VolumeConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Decod
 	if r.TryDecodeAsNil() {
 		x.PersistentVolumeRecyclerConfiguration = PersistentVolumeRecyclerConfiguration{}
 	} else {
-		yyv9 := &x.PersistentVolumeRecyclerConfiguration
+		yyv11 := &x.PersistentVolumeRecyclerConfiguration
-		yyv9.CodecDecodeSelf(d)
+		yyv11.CodecDecodeSelf(d)
 	}
-	yyj7++
+	yyj8++
-	if yyhl7 {
+	if yyhl8 {
-		yyb7 = yyj7 > l
+		yyb8 = yyj8 > l
 	} else {
-		yyb7 = r.CheckBreak()
+		yyb8 = r.CheckBreak()
 	}
-	if yyb7 {
+	if yyb8 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -9123,17 +9164,17 @@ func (x *VolumeConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Decod
 		x.FlexVolumePluginDir = string(r.DecodeString())
 	}
 	for {
-		yyj7++
+		yyj8++
-		if yyhl7 {
+		if yyhl8 {
-			yyb7 = yyj7 > l
+			yyb8 = yyj8 > l
 		} else {
-			yyb7 = r.CheckBreak()
+			yyb8 = r.CheckBreak()
 		}
-		if yyb7 {
+		if yyb8 {
 			break
 		}
 		z.DecSendContainerState(codecSelfer_containerArrayElem1234)
-		z.DecStructFieldNotFound(yyj7-1, "")
+		z.DecStructFieldNotFound(yyj8-1, "")
 	}
 	z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 }

pkg/apis/componentconfig/types.go

@@ -577,6 +577,9 @@ type VolumeConfiguration struct {
 	// provisioning is not supported in any way, won't work in a multi-node cluster, and
 	// should not be used for anything other than testing or development.
 	EnableHostPathProvisioning bool `json:"enableHostPathProvisioning"`
+	// enableDynamicProvisioning enables the provisioning of volumes when running within an environment
+	// that supports dynamic provisioning. Defaults to true.
+	EnableDynamicProvisioning bool `json:"enableDynamicProvisioning"`
 	// persistentVolumeRecyclerConfiguration holds configuration for persistent volume plugins.
 	PersistentVolumeRecyclerConfiguration PersistentVolumeRecyclerConfiguration `json:"persitentVolumeRecyclerConfiguration"`
 	// volumePluginDir is the full path of the directory in which the flex

pkg/controller/persistentvolume/controller.go

@@ -106,18 +106,19 @@ const createProvisionedPVInterval = 10 * time.Second
 // framework.Controllers that watch PersistentVolume and PersistentVolumeClaim
 // changes.
 type PersistentVolumeController struct {
-	volumeController       *framework.Controller
+	volumeController          *framework.Controller
-	volumeControllerStopCh chan struct{}
+	volumeControllerStopCh    chan struct{}
-	volumeSource           cache.ListerWatcher
+	volumeSource              cache.ListerWatcher
-	claimController        *framework.Controller
+	claimController           *framework.Controller
-	claimControllerStopCh  chan struct{}
+	claimControllerStopCh     chan struct{}
-	claimSource            cache.ListerWatcher
+	claimSource               cache.ListerWatcher
-	kubeClient             clientset.Interface
+	kubeClient                clientset.Interface
-	eventRecorder          record.EventRecorder
+	eventRecorder             record.EventRecorder
-	cloud                  cloudprovider.Interface
+	cloud                     cloudprovider.Interface
-	recyclePluginMgr       vol.VolumePluginMgr
+	recyclePluginMgr          vol.VolumePluginMgr
-	provisioner            vol.ProvisionableVolumePlugin
+	provisioner               vol.ProvisionableVolumePlugin
-	clusterName            string
+	enableDynamicProvisioning bool
+	clusterName               string
 
 	// Cache of the last known version of volumes and claims. This cache is
 	// thread safe as long as the volumes/claims there are not modified, they
@@ -1062,8 +1063,11 @@ func (ctrl *PersistentVolumeController) doDeleteVolume(volume *api.PersistentVol
 	return nil
 }
 
-// provisionClaim starts new asynchronous operation to provision a claim.
+// provisionClaim starts new asynchronous operation to provision a claim if provisioning is enabled.
 func (ctrl *PersistentVolumeController) provisionClaim(claim *api.PersistentVolumeClaim) error {
+	if !ctrl.enableDynamicProvisioning {
+		return nil
+	}
 	glog.V(4).Infof("provisionClaim[%s]: started", claimToClaimKey(claim))
 	opName := fmt.Sprintf("provision-%s[%s]", claimToClaimKey(claim), string(claim.UID))
 	ctrl.scheduleOperation(opName, func() error {

pkg/controller/persistentvolume/controller_base.go

@@ -52,6 +52,7 @@ func NewPersistentVolumeController(
 	clusterName string,
 	volumeSource, claimSource cache.ListerWatcher,
 	eventRecorder record.EventRecorder,
+	enableDynamicProvisioning bool,
 ) *PersistentVolumeController {
 
 	if eventRecorder == nil {
@@ -68,6 +69,7 @@ func NewPersistentVolumeController(
 		runningOperations:             goroutinemap.NewGoRoutineMap(),
 		cloud:                         cloud,
 		provisioner:                   provisioner,
+		enableDynamicProvisioning:     enableDynamicProvisioning,
 		clusterName:                   clusterName,
 		createProvisionedPVRetryCount: createProvisionedPVRetryCount,
 		createProvisionedPVInterval:   createProvisionedPVInterval,

pkg/controller/persistentvolume/controller_test.go

@@ -164,7 +164,7 @@ func TestControllerSync(t *testing.T) {
 		client := &fake.Clientset{}
 		volumeSource := framework.NewFakeControllerSource()
 		claimSource := framework.NewFakeControllerSource()
-		ctrl := newTestController(client, volumeSource, claimSource)
+		ctrl := newTestController(client, volumeSource, claimSource, true)
 		reactor := newVolumeReactor(client, ctrl, volumeSource, claimSource, test.errors)
 		for _, claim := range test.initialClaims {
 			claimSource.Add(claim)

pkg/controller/persistentvolume/framework_test.go

@@ -555,7 +555,7 @@ func newVolumeReactor(client *fake.Clientset, ctrl *PersistentVolumeController,
 	return reactor
 }
 
-func newTestController(kubeClient clientset.Interface, volumeSource, claimSource cache.ListerWatcher) *PersistentVolumeController {
+func newTestController(kubeClient clientset.Interface, volumeSource, claimSource cache.ListerWatcher, enableDynamicProvisioning bool) *PersistentVolumeController {
 	if volumeSource == nil {
 		volumeSource = framework.NewFakeControllerSource()
 	}
@@ -572,6 +572,7 @@ func newTestController(kubeClient clientset.Interface, volumeSource, claimSource
 		volumeSource,
 		claimSource,
 		record.NewFakeRecorder(1000), // event recorder
+		enableDynamicProvisioning,
 	)
 
 	// Speed up the test
@@ -830,7 +831,7 @@ func runSyncTests(t *testing.T, tests []controllerTest) {
 
 		// Initialize the controller
 		client := &fake.Clientset{}
-		ctrl := newTestController(client, nil, nil)
+		ctrl := newTestController(client, nil, nil, true)
 		reactor := newVolumeReactor(client, ctrl, nil, nil, test.errors)
 		for _, claim := range test.initialClaims {
 			ctrl.claims.Add(claim)
@@ -874,7 +875,7 @@ func runMultisyncTests(t *testing.T, tests []controllerTest) {
 
 		// Initialize the controller
 		client := &fake.Clientset{}
-		ctrl := newTestController(client, nil, nil)
+		ctrl := newTestController(client, nil, nil, true)
 		reactor := newVolumeReactor(client, ctrl, nil, nil, test.errors)
 		for _, claim := range test.initialClaims {
 			ctrl.claims.Add(claim)

pkg/controller/persistentvolume/options/options.go

@@ -35,6 +35,7 @@ type VolumeConfigFlags struct {
 	PersistentVolumeRecyclerMinimumTimeoutHostPath      int
 	PersistentVolumeRecyclerIncrementTimeoutHostPath    int
 	EnableHostPathProvisioning                          bool
+	EnableDynamicProvisioning                           bool
 }
 
 type PersistentVolumeControllerOptions struct {
@@ -53,6 +54,7 @@ func NewPersistentVolumeControllerOptions() PersistentVolumeControllerOptions {
 			PersistentVolumeRecyclerMinimumTimeoutHostPath:   60,
 			PersistentVolumeRecyclerIncrementTimeoutHostPath: 30,
 			EnableHostPathProvisioning:                       false,
+			EnableDynamicProvisioning:                        true,
 		},
 	}
 }
@@ -84,4 +86,6 @@ func (o *PersistentVolumeControllerOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&o.VolumeConfigFlags.EnableHostPathProvisioning, "enable-hostpath-provisioner", o.VolumeConfigFlags.EnableHostPathProvisioning,
 		"Enable HostPath PV provisioning when running without a cloud provider. This allows testing and development of provisioning features. "+
 			"HostPath provisioning is not supported in any way, won't work in a multi-node cluster, and should not be used for anything other than testing or development.")
+	fs.BoolVar(&o.VolumeConfigFlags.EnableDynamicProvisioning, "enable-dynamic-provisioning", o.VolumeConfigFlags.EnableDynamicProvisioning,
+		"Enable dynamic provisioning for environments that support it.")
 }

pkg/controller/persistentvolume/provision_test.go

@@ -254,3 +254,12 @@ func TestProvisionMultiSync(t *testing.T) {
 
 	runMultisyncTests(t, tests)
 }
+
+// When provisioning is disabled, provisioning a claim should instantly return nil
+func TestDisablingDynamicProvisioner(t *testing.T) {
+	ctrl := newTestController(nil, nil, nil, false)
+	retVal := ctrl.provisionClaim(nil)
+	if retVal != nil {
+		t.Errorf("Expected nil return but got %v", retVal)
+	}
+}

test/integration/persistent_volumes_test.go

@@ -678,7 +678,7 @@ func createClients(t *testing.T, s *httptest.Server) (*clientset.Clientset, *per
 	cloud := &fake_cloud.FakeCloud{}
 
 	syncPeriod := getSyncPeriod()
-	ctrl := persistentvolumecontroller.NewPersistentVolumeController(binderClient, syncPeriod, plugin, plugins, cloud, "", nil, nil, nil)
+	ctrl := persistentvolumecontroller.NewPersistentVolumeController(binderClient, syncPeriod, plugin, plugins, cloud, "", nil, nil, nil, true)
 
 	watchPV, err := testClient.PersistentVolumes().Watch(api.ListOptions{})
 	if err != nil {