combine feature gate VolumeScheduling and DynamicProvisioningScheduling into one

This commit is contained in:
lichuqiang 2018-08-15 12:14:19 +08:00
parent cd06419973
commit b4a57f6855
12 changed files with 39 additions and 66 deletions

View File

@ -3137,7 +3137,7 @@ func ValidateTopologySelectorTerm(term core.TopologySelectorTerm, fldPath *field
exprMap := make(map[string]sets.String) exprMap := make(map[string]sets.String)
exprPath := fldPath.Child("matchLabelExpressions") exprPath := fldPath.Child("matchLabelExpressions")
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) { if utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
// Allow empty MatchLabelExpressions, in case this field becomes optional in the future. // Allow empty MatchLabelExpressions, in case this field becomes optional in the future.
for i, req := range term.MatchLabelExpressions { for i, req := range term.MatchLabelExpressions {
@ -3152,7 +3152,7 @@ func ValidateTopologySelectorTerm(term core.TopologySelectorTerm, fldPath *field
exprMap[req.Key] = valueSet exprMap[req.Key] = valueSet
} }
} else if len(term.MatchLabelExpressions) != 0 { } else if len(term.MatchLabelExpressions) != 0 {
allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate DynamicProvisioningScheduling")) allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate VolumeScheduling"))
} }
return exprMap, allErrs return exprMap, allErrs

View File

@ -68,16 +68,14 @@ type StorageClass struct {
// VolumeBindingMode indicates how PersistentVolumeClaims should be // VolumeBindingMode indicates how PersistentVolumeClaims should be
// provisioned and bound. When unset, VolumeBindingImmediate is used. // provisioned and bound. When unset, VolumeBindingImmediate is used.
// This field is alpha-level and is only honored by servers that enable // This field is only honored by servers that enable the VolumeScheduling feature.
// the VolumeScheduling feature.
// +optional // +optional
VolumeBindingMode *VolumeBindingMode VolumeBindingMode *VolumeBindingMode
// Restrict the node topologies where volumes can be dynamically provisioned. // Restrict the node topologies where volumes can be dynamically provisioned.
// Each volume plugin defines its own supported topology specifications. // Each volume plugin defines its own supported topology specifications.
// An empty TopologySelectorTerm list means there is no topology restriction. // An empty TopologySelectorTerm list means there is no topology restriction.
// This field is alpha-level and is only honored by servers that enable // This field is only honored by servers that enable the VolumeScheduling feature.
// the DynamicProvisioningScheduling feature.
// +optional // +optional
AllowedTopologies []api.TopologySelectorTerm AllowedTopologies []api.TopologySelectorTerm
} }

View File

@ -26,8 +26,6 @@ import (
func DropDisabledAlphaFields(class *storage.StorageClass) { func DropDisabledAlphaFields(class *storage.StorageClass) {
if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) { if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
class.VolumeBindingMode = nil class.VolumeBindingMode = nil
}
if !utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) {
class.AllowedTopologies = nil class.AllowedTopologies = nil
} }
} }

View File

@ -250,8 +250,8 @@ func validateAllowedTopologies(topologies []api.TopologySelectorTerm, fldPath *f
return allErrs return allErrs
} }
if !utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) { if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate DynamicProvisioningScheduling")) allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate VolumeScheduling"))
} }
rawTopologies := make([]map[string]sets.String, len(topologies)) rawTopologies := make([]map[string]sets.String, len(topologies))

View File

@ -140,7 +140,7 @@ const annStorageProvisioner = "volume.beta.kubernetes.io/storage-provisioner"
// This annotation is added to a PVC that has been triggered by scheduler to // This annotation is added to a PVC that has been triggered by scheduler to
// be dynamically provisioned. Its value is the name of the selected node. // be dynamically provisioned. Its value is the name of the selected node.
const annSelectedNode = "volume.alpha.kubernetes.io/selected-node" const annSelectedNode = "volume.kubernetes.io/selected-node"
// If the provisioner name in a storage class is set to "kubernetes.io/no-provisioner", // If the provisioner name in a storage class is set to "kubernetes.io/no-provisioner",
// then dynamic provisioning is not supported by the storage. // then dynamic provisioning is not supported by the storage.
@ -290,14 +290,12 @@ func (ctrl *PersistentVolumeController) shouldDelayBinding(claim *v1.PersistentV
return false, nil return false, nil
} }
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) { // When feature VolumeScheduling enabled,
// When feature DynamicProvisioningScheduling enabled, // Scheduler signal to the PV controller to start dynamic
// Scheduler signal to the PV controller to start dynamic // provisioning by setting the "annSelectedNode" annotation
// provisioning by setting the "annSelectedNode" annotation // in the PVC
// in the PVC if _, ok := claim.Annotations[annSelectedNode]; ok {
if _, ok := claim.Annotations[annSelectedNode]; ok { return false, nil
return false, nil
}
} }
className := v1helper.GetPersistentVolumeClaimClass(claim) className := v1helper.GetPersistentVolumeClaimClass(claim)
@ -1477,25 +1475,22 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claim *v1.Persis
} }
var selectedNode *v1.Node = nil var selectedNode *v1.Node = nil
var allowedTopologies []v1.TopologySelectorTerm = nil if nodeName, ok := claim.Annotations[annSelectedNode]; ok {
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) { selectedNode, err = ctrl.NodeLister.Get(nodeName)
if nodeName, ok := claim.Annotations[annSelectedNode]; ok { if err != nil {
selectedNode, err = ctrl.NodeLister.Get(nodeName) strerr := fmt.Sprintf("Failed to get target node: %v", err)
if err != nil { glog.V(3).Infof("unexpected error getting target node %q for claim %q: %v", nodeName, claimToClaimKey(claim), err)
strerr := fmt.Sprintf("Failed to get target node: %v", err) ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, events.ProvisioningFailed, strerr)
glog.V(3).Infof("unexpected error getting target node %q for claim %q: %v", nodeName, claimToClaimKey(claim), err) return pluginName, err
ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, events.ProvisioningFailed, strerr)
return pluginName, err
}
} }
allowedTopologies = storageClass.AllowedTopologies
} }
allowedTopologies := storageClass.AllowedTopologies
opComplete := util.OperationCompleteHook(plugin.GetPluginName(), "volume_provision") opComplete := util.OperationCompleteHook(plugin.GetPluginName(), "volume_provision")
volume, err = provisioner.Provision(selectedNode, allowedTopologies) volume, err = provisioner.Provision(selectedNode, allowedTopologies)
opComplete(&err) opComplete(&err)
if err != nil { if err != nil {
// Other places of failure have nothing to do with DynamicProvisioningScheduling, // Other places of failure have nothing to do with VolumeScheduling,
// so just let controller retry in the next sync. We'll only call func // so just let controller retry in the next sync. We'll only call func
// rescheduleProvisioning here when the underlying provisioning actually failed. // rescheduleProvisioning here when the underlying provisioning actually failed.
ctrl.rescheduleProvisioning(claim) ctrl.rescheduleProvisioning(claim)

View File

@ -456,7 +456,7 @@ func TestAssumeUpdatePVCCache(t *testing.T) {
// Assume PVC // Assume PVC
newPVC := pvc.DeepCopy() newPVC := pvc.DeepCopy()
newPVC.Annotations["volume.alpha.kubernetes.io/selected-node"] = "test-node" newPVC.Annotations[annSelectedNode] = "test-node"
if err := cache.Assume(newPVC); err != nil { if err := cache.Assume(newPVC); err != nil {
t.Fatalf("failed to assume PVC: %v", err) t.Fatalf("failed to assume PVC: %v", err)
} }

View File

@ -25,12 +25,10 @@ import (
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/labels"
utilfeature "k8s.io/apiserver/pkg/util/feature"
coreinformers "k8s.io/client-go/informers/core/v1" coreinformers "k8s.io/client-go/informers/core/v1"
storageinformers "k8s.io/client-go/informers/storage/v1" storageinformers "k8s.io/client-go/informers/storage/v1"
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
"k8s.io/kubernetes/pkg/features"
volumeutil "k8s.io/kubernetes/pkg/volume/util" volumeutil "k8s.io/kubernetes/pkg/volume/util"
) )
@ -169,13 +167,11 @@ func (b *volumeBinder) FindPodVolumes(pod *v1.Pod, node *v1.Node) (unboundVolume
return false, false, err return false, false, err
} }
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) { // Try to provision for unbound volumes
// Try to provision for unbound volumes if !unboundVolumesSatisfied {
if !unboundVolumesSatisfied { unboundVolumesSatisfied, err = b.checkVolumeProvisions(pod, claimsToProvision, node)
unboundVolumesSatisfied, err = b.checkVolumeProvisions(pod, claimsToProvision, node) if err != nil {
if err != nil { return false, false, err
return false, false, err
}
} }
} }
} }

View File

@ -312,12 +312,6 @@ const (
// Support Pod Ready++ // Support Pod Ready++
PodReadinessGates utilfeature.Feature = "PodReadinessGates" PodReadinessGates utilfeature.Feature = "PodReadinessGates"
// owner: @lichuqiang
// alpha: v1.11
//
// Extend the default scheduler to be aware of volume topology and handle PV provisioning
DynamicProvisioningScheduling utilfeature.Feature = "DynamicProvisioningScheduling"
// owner: @kevtaylor // owner: @kevtaylor
// alpha: v1.11 // alpha: v1.11
// //
@ -415,7 +409,6 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
RunAsGroup: {Default: false, PreRelease: utilfeature.Alpha}, RunAsGroup: {Default: false, PreRelease: utilfeature.Alpha},
VolumeSubpath: {Default: true, PreRelease: utilfeature.GA}, VolumeSubpath: {Default: true, PreRelease: utilfeature.GA},
BalanceAttachedNodeVolumes: {Default: false, PreRelease: utilfeature.Alpha}, BalanceAttachedNodeVolumes: {Default: false, PreRelease: utilfeature.Alpha},
DynamicProvisioningScheduling: {Default: false, PreRelease: utilfeature.Alpha},
PodReadinessGates: {Default: true, PreRelease: utilfeature.Beta}, PodReadinessGates: {Default: true, PreRelease: utilfeature.Beta},
VolumeSubpathEnvExpansion: {Default: false, PreRelease: utilfeature.Alpha}, VolumeSubpathEnvExpansion: {Default: false, PreRelease: utilfeature.Alpha},
KubeletPluginsWatcher: {Default: false, PreRelease: utilfeature.Alpha}, KubeletPluginsWatcher: {Default: false, PreRelease: utilfeature.Alpha},

View File

@ -345,7 +345,7 @@ func SelectZonesForVolume(zoneParameterPresent, zonesParameterPresent bool, zone
var zoneFromNode string var zoneFromNode string
// pick one zone from node if present // pick one zone from node if present
if node != nil { if node != nil {
// DynamicProvisioningScheduling implicit since node is not nil // VolumeScheduling implicit since node is not nil
if zoneParameterPresent || zonesParameterPresent { if zoneParameterPresent || zonesParameterPresent {
return nil, fmt.Errorf("zone[s] cannot be specified in StorageClass if VolumeBindingMode is set to WaitForFirstConsumer. Please specify allowedTopologies in StorageClass for constraining zones") return nil, fmt.Errorf("zone[s] cannot be specified in StorageClass if VolumeBindingMode is set to WaitForFirstConsumer. Please specify allowedTopologies in StorageClass for constraining zones")
} }
@ -373,7 +373,7 @@ func SelectZonesForVolume(zoneParameterPresent, zonesParameterPresent bool, zone
} }
if allowedZones.Len() > 0 { if allowedZones.Len() > 0 {
// DynamicProvisioningScheduling implicit since allowedZones present // VolumeScheduling implicit since allowedZones present
if zoneParameterPresent || zonesParameterPresent { if zoneParameterPresent || zonesParameterPresent {
return nil, fmt.Errorf("zone[s] cannot be specified in StorageClass if allowedTopologies specified") return nil, fmt.Errorf("zone[s] cannot be specified in StorageClass if allowedTopologies specified")
} }

View File

@ -487,16 +487,13 @@ func ClusterRoles() []rbacv1.ClusterRole {
} }
if utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) { if utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
rules := []rbacv1.PolicyRule{
rbacv1helpers.NewRule(ReadUpdate...).Groups(legacyGroup).Resources("persistentvolumes").RuleOrDie(),
rbacv1helpers.NewRule(Read...).Groups(storageGroup).Resources("storageclasses").RuleOrDie(),
}
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) {
rules = append(rules, rbacv1helpers.NewRule(ReadUpdate...).Groups(legacyGroup).Resources("persistentvolumeclaims").RuleOrDie())
}
roles = append(roles, rbacv1.ClusterRole{ roles = append(roles, rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: "system:volume-scheduler"}, ObjectMeta: metav1.ObjectMeta{Name: "system:volume-scheduler"},
Rules: rules, Rules: []rbacv1.PolicyRule{
rbacv1helpers.NewRule(ReadUpdate...).Groups(legacyGroup).Resources("persistentvolumes").RuleOrDie(),
rbacv1helpers.NewRule(Read...).Groups(storageGroup).Resources("storageclasses").RuleOrDie(),
rbacv1helpers.NewRule(ReadUpdate...).Groups(legacyGroup).Resources("persistentvolumeclaims").RuleOrDie(),
},
}) })
} }

View File

@ -62,16 +62,14 @@ type StorageClass struct {
// VolumeBindingMode indicates how PersistentVolumeClaims should be // VolumeBindingMode indicates how PersistentVolumeClaims should be
// provisioned and bound. When unset, VolumeBindingImmediate is used. // provisioned and bound. When unset, VolumeBindingImmediate is used.
// This field is alpha-level and is only honored by servers that enable // This field is only honored by servers that enable the VolumeScheduling feature.
// the VolumeScheduling feature.
// +optional // +optional
VolumeBindingMode *VolumeBindingMode `json:"volumeBindingMode,omitempty" protobuf:"bytes,7,opt,name=volumeBindingMode"` VolumeBindingMode *VolumeBindingMode `json:"volumeBindingMode,omitempty" protobuf:"bytes,7,opt,name=volumeBindingMode"`
// Restrict the node topologies where volumes can be dynamically provisioned. // Restrict the node topologies where volumes can be dynamically provisioned.
// Each volume plugin defines its own supported topology specifications. // Each volume plugin defines its own supported topology specifications.
// An empty TopologySelectorTerm list means there is no topology restriction. // An empty TopologySelectorTerm list means there is no topology restriction.
// This field is alpha-level and is only honored by servers that enable // This field is only honored by servers that enable the VolumeScheduling feature.
// the DynamicProvisioningScheduling feature.
// +optional // +optional
AllowedTopologies []v1.TopologySelectorTerm `json:"allowedTopologies,omitempty" protobuf:"bytes,8,rep,name=allowedTopologies"` AllowedTopologies []v1.TopologySelectorTerm `json:"allowedTopologies,omitempty" protobuf:"bytes,8,rep,name=allowedTopologies"`
} }

View File

@ -62,16 +62,14 @@ type StorageClass struct {
// VolumeBindingMode indicates how PersistentVolumeClaims should be // VolumeBindingMode indicates how PersistentVolumeClaims should be
// provisioned and bound. When unset, VolumeBindingImmediate is used. // provisioned and bound. When unset, VolumeBindingImmediate is used.
// This field is alpha-level and is only honored by servers that enable // This field is only honored by servers that enable the VolumeScheduling feature.
// the VolumeScheduling feature.
// +optional // +optional
VolumeBindingMode *VolumeBindingMode `json:"volumeBindingMode,omitempty" protobuf:"bytes,7,opt,name=volumeBindingMode"` VolumeBindingMode *VolumeBindingMode `json:"volumeBindingMode,omitempty" protobuf:"bytes,7,opt,name=volumeBindingMode"`
// Restrict the node topologies where volumes can be dynamically provisioned. // Restrict the node topologies where volumes can be dynamically provisioned.
// Each volume plugin defines its own supported topology specifications. // Each volume plugin defines its own supported topology specifications.
// An empty TopologySelectorTerm list means there is no topology restriction. // An empty TopologySelectorTerm list means there is no topology restriction.
// This field is alpha-level and is only honored by servers that enable // This field is only honored by servers that enable the VolumeScheduling feature.
// the DynamicProvisioningScheduling feature.
// +optional // +optional
AllowedTopologies []v1.TopologySelectorTerm `json:"allowedTopologies,omitempty" protobuf:"bytes,8,rep,name=allowedTopologies"` AllowedTopologies []v1.TopologySelectorTerm `json:"allowedTopologies,omitempty" protobuf:"bytes,8,rep,name=allowedTopologies"`
} }