github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-09-26 04:36:00 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #105609 from pohly/generic-ephemeral-volume-ga

Browse Source 
generic ephemeral volume GA
This commit is contained in:
Kubernetes Prow Robot
2021-10-28 17:36:50 -07:00
committed by GitHub
parent 8e37a3b324 d55f7c85ef
commit c592bd40f2
37 changed files with 175 additions and 570 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/scheduler/framework/plugins/feature/feature.go
View File

@@ -26,5 +26,4 @@ type Features struct {
EnableReadWriteOncePod bool
EnableVolumeCapacityPriority bool
EnableCSIStorageCapacity bool
EnableGenericEphemeralVolume bool
}

35
pkg/scheduler/framework/plugins/nodevolumelimits/csi.go
View File

@@ -22,11 +22,11 @@ import (
v1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/rand"
corelisters "k8s.io/client-go/listers/core/v1"
storagelisters "k8s.io/client-go/listers/storage/v1"
ephemeral "k8s.io/component-helpers/storage/ephemeral"
storagehelpers "k8s.io/component-helpers/storage/volume"
csitrans "k8s.io/csi-translation-lib"
"k8s.io/klog/v2"
@@ -56,8 +56,6 @@ type CSILimits struct {
randomVolumeIDPrefix string
translator InTreeToCSITranslator
enableGenericEphemeralVolume bool
}
var _ framework.FilterPlugin = &CSILimits{}
@@ -152,23 +150,17 @@ func (pl *CSILimits) filterAttachableVolumes(
for _, vol := range pod.Spec.Volumes {
// CSI volumes can only be used through a PVC.
pvcName := ""
ephemeral := false
isEphemeral := false
switch {
case vol.PersistentVolumeClaim != nil:
pvcName = vol.PersistentVolumeClaim.ClaimName
case vol.Ephemeral != nil:
if newPod && !pl.enableGenericEphemeralVolume {
return fmt.Errorf(
"volume %s is a generic ephemeral volume, but that feature is disabled in kube-scheduler",
vol.Name,
)
}
// Generic ephemeral inline volumes also use a PVC,
// just with a computed name and certain ownership.
// That is checked below once the pvc object is
// retrieved.
pvcName = pod.Name + "-" + vol.Name
ephemeral = true
pvcName = ephemeral.VolumeClaimName(pod, &vol)
isEphemeral = true
default:
continue
}
@@ -193,8 +185,10 @@ func (pl *CSILimits) filterAttachableVolumes(
}
// The PVC for an ephemeral volume must be owned by the pod.
if ephemeral && !metav1.IsControlledBy(pvc, pod) {
return fmt.Errorf("PVC %s/%s is not owned by pod", pod.Namespace, pvcName)
if isEphemeral {
if err := ephemeral.VolumeIsForPod(pod, pvc); err != nil {
return err
}
}
driverName, volumeHandle := pl.getCSIDriverInfo(csiNode, pvc)
@@ -316,13 +310,12 @@ func NewCSI(_ runtime.Object, handle framework.Handle, fts feature.Features) (fr
scLister := informerFactory.Storage().V1().StorageClasses().Lister()
return &CSILimits{
csiNodeLister: csiNodesLister,
pvLister: pvLister,
pvcLister: pvcLister,
scLister: scLister,
randomVolumeIDPrefix: rand.String(32),
translator: csitrans.New(),
enableGenericEphemeralVolume: fts.EnableGenericEphemeralVolume,
csiNodeLister: csiNodesLister,
pvLister: pvLister,
pvcLister: pvcLister,
scLister: scLister,
randomVolumeIDPrefix: rand.String(32),
translator: csitrans.New(),
}, nil
}

11
pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go
View File

@@ -546,13 +546,6 @@ func TestCSILimits(t *testing.T) {
test: "should not count in-tree and count csi volumes if migration is disabled (when scheduling in-tree volumes)",
},
// ephemeral volumes
{
newPod: ephemeralVolumePod,
filterName: "csi",
driverNames: []string{ebsCSIDriverName},
test: "ephemeral volume feature disabled",
wantStatus: framework.NewStatus(framework.Error, "volume xyz is a generic ephemeral volume, but that feature is disabled in kube-scheduler"),
},
{
newPod: ephemeralVolumePod,
filterName: "csi",
@@ -568,7 +561,7 @@ func TestCSILimits(t *testing.T) {
extraClaims: []v1.PersistentVolumeClaim{*conflictingClaim},
driverNames: []string{ebsCSIDriverName},
test: "ephemeral volume not owned",
wantStatus: framework.NewStatus(framework.Error, "PVC test/abc-xyz is not owned by pod"),
wantStatus: framework.NewStatus(framework.Error, "PVC test/abc-xyz was not created for pod test/abc (pod is not owner)"),
},
{
newPod: ephemeralVolumePod,
@@ -657,8 +650,6 @@ func TestCSILimits(t *testing.T) {
scLister: getFakeCSIStorageClassLister(scName, test.driverNames[0]),
randomVolumeIDPrefix: rand.String(32),
translator: csitrans.New(),
enableGenericEphemeralVolume: test.ephemeralEnabled,
}
gotStatus := p.Filter(context.Background(), nil, test.newPod, node)
if !reflect.DeepEqual(gotStatus, test.wantStatus) {

24
pkg/scheduler/framework/plugins/nodevolumelimits/non_csi.go
View File

@@ -26,13 +26,13 @@ import (
v1 "k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/informers"
corelisters "k8s.io/client-go/listers/core/v1"
storagelisters "k8s.io/client-go/listers/storage/v1"
"k8s.io/component-helpers/storage/ephemeral"
csilibplugins "k8s.io/csi-translation-lib/plugins"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/scheduler/framework"
@@ -117,8 +117,6 @@ type nonCSILimits struct {
// It is used to prefix volumeID generated inside the predicate() method to
// avoid conflicts with any real volume.
randomVolumeIDPrefix string
enableGenericEphemeralVolume bool
}
var _ framework.FilterPlugin = &nonCSILimits{}
@@ -191,8 +189,6 @@ func newNonCSILimits(
pvcLister: pvcLister,
scLister: scLister,
randomVolumeIDPrefix: rand.String(32),
enableGenericEphemeralVolume: fts.EnableGenericEphemeralVolume,
}
return pl
@@ -288,23 +284,17 @@ func (pl *nonCSILimits) filterVolumes(pod *v1.Pod, newPod bool, filteredVolumes
}
pvcName := ""
ephemeral := false
isEphemeral := false
switch {
case vol.PersistentVolumeClaim != nil:
pvcName = vol.PersistentVolumeClaim.ClaimName
case vol.Ephemeral != nil:
if !pl.enableGenericEphemeralVolume {
return fmt.Errorf(
"volume %s is a generic ephemeral volume, but that feature is disabled in kube-scheduler",
vol.Name,
)
}
// Generic ephemeral inline volumes also use a PVC,
// just with a computed name and certain ownership.
// That is checked below once the pvc object is
// retrieved.
pvcName = pod.Name + "-" + vol.Name
ephemeral = true
pvcName = ephemeral.VolumeClaimName(pod, vol)
isEphemeral = true
default:
continue
}
@@ -332,8 +322,10 @@ func (pl *nonCSILimits) filterVolumes(pod *v1.Pod, newPod bool, filteredVolumes
}
// The PVC for an ephemeral volume must be owned by the pod.
if ephemeral && !metav1.IsControlledBy(pvc, pod) {
return fmt.Errorf("PVC %s/%s is not owned by pod", pod.Namespace, pvcName)
if isEphemeral {
if err := ephemeral.VolumeIsForPod(pod, pvc); err != nil {
return err
}
}
pvName := pvc.Spec.VolumeName

11
pkg/scheduler/framework/plugins/nodevolumelimits/non_csi_test.go
View File

@@ -86,11 +86,6 @@ func TestEphemeralLimits(t *testing.T) {
test string
wantStatus *framework.Status
}{
{
newPod: ephemeralVolumePod,
test: "volume feature disabled",
wantStatus: framework.NewStatus(framework.Error, "volume xyz is a generic ephemeral volume, but that feature is disabled in kube-scheduler"),
},
{
newPod: ephemeralVolumePod,
ephemeralEnabled: true,
@@ -102,7 +97,7 @@ func TestEphemeralLimits(t *testing.T) {
ephemeralEnabled: true,
extraClaims: []v1.PersistentVolumeClaim{*conflictingClaim},
test: "volume not owned",
wantStatus: framework.NewStatus(framework.Error, "PVC test/abc-xyz is not owned by pod"),
wantStatus: framework.NewStatus(framework.Error, "PVC test/abc-xyz was not created for pod test/abc (pod is not owner)"),
},
{
newPod: ephemeralVolumePod,
@@ -123,9 +118,7 @@ func TestEphemeralLimits(t *testing.T) {
for _, test := range tests {
t.Run(test.test, func(t *testing.T) {
fts := feature.Features{
EnableGenericEphemeralVolume: test.ephemeralEnabled,
}
fts := feature.Features{}
node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, int64(test.maxVols), filterName)
p := newNonCSILimits(filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(filterName, driverName), getFakePVLister(filterName), append(getFakePVCLister(filterName), test.extraClaims...), fts).(framework.FilterPlugin)
gotStatus := p.Filter(context.Background(), nil, test.newPod, node)

1
pkg/scheduler/framework/plugins/registry.go
View File

@@ -51,7 +51,6 @@ func NewInTreeRegistry() runtime.Registry {
EnableReadWriteOncePod: feature.DefaultFeatureGate.Enabled(features.ReadWriteOncePod),
EnableVolumeCapacityPriority: feature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority),
EnableCSIStorageCapacity: feature.DefaultFeatureGate.Enabled(features.CSIStorageCapacity),
EnableGenericEphemeralVolume: feature.DefaultFeatureGate.Enabled(features.GenericEphemeralVolume),
}
return runtime.Registry{

19
pkg/scheduler/framework/plugins/volumebinding/binder.go
View File

@@ -40,6 +40,7 @@ import (
corelisters "k8s.io/client-go/listers/core/v1"
storagelisters "k8s.io/client-go/listers/storage/v1"
storagelistersv1beta1 "k8s.io/client-go/listers/storage/v1beta1"
"k8s.io/component-helpers/storage/ephemeral"
storagehelpers "k8s.io/component-helpers/storage/volume"
csitrans "k8s.io/csi-translation-lib"
csiplugins "k8s.io/csi-translation-lib/plugins"
@@ -686,29 +687,25 @@ func (b *volumeBinder) checkBindings(pod *v1.Pod, bindings []*BindingInfo, claim
func (b *volumeBinder) isVolumeBound(pod *v1.Pod, vol *v1.Volume) (bound bool, pvc *v1.PersistentVolumeClaim, err error) {
pvcName := ""
ephemeral := false
isEphemeral := false
switch {
case vol.PersistentVolumeClaim != nil:
pvcName = vol.PersistentVolumeClaim.ClaimName
case vol.Ephemeral != nil:
if !utilfeature.DefaultFeatureGate.Enabled(features.GenericEphemeralVolume) {
return false, nil, fmt.Errorf(
"volume %s is a generic ephemeral volume, but that feature is disabled in kube-scheduler",
vol.Name,
)
}
// Generic ephemeral inline volumes also use a PVC,
// just with a computed name, and...
pvcName = pod.Name + "-" + vol.Name
ephemeral = true
pvcName = ephemeral.VolumeClaimName(pod, vol)
isEphemeral = true
default:
return true, nil, nil
}
bound, pvc, err = b.isPVCBound(pod.Namespace, pvcName)
// ... the PVC must be owned by the pod.
if ephemeral && err == nil && pvc != nil && !metav1.IsControlledBy(pvc, pod) {
return false, nil, fmt.Errorf("PVC %s/%s is not owned by pod", pod.Namespace, pvcName)
if isEphemeral && err == nil && pvc != nil {
if err := ephemeral.VolumeIsForPod(pod, pvc); err != nil {
return false, nil, err
}
}
return
}

18
pkg/scheduler/framework/plugins/volumebinding/binder_test.go
View File

@@ -833,9 +833,6 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) {
// If nil, makePod with podPVCs
pod *v1.Pod
// GenericEphemeralVolume feature enabled?
ephemeral bool
// Expected podBindingCache fields
expectedBindings []*BindingInfo
@@ -942,7 +939,6 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) {
withNamespace("testns").
withNodeName("node1").
withGenericEphemeralVolume("no-such-pvc").Pod,
ephemeral: true,
shouldFail: true,
},
"generic-ephemeral,with-pvc": {
@@ -952,7 +948,6 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) {
withGenericEphemeralVolume("test-volume").Pod,
cachePVCs: []*v1.PersistentVolumeClaim{correctGenericPVC},
pvs: []*v1.PersistentVolume{pvBoundGeneric},
ephemeral: true,
},
"generic-ephemeral,wrong-pvc": {
pod: makePod("test-pod").
@@ -961,17 +956,6 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) {
withGenericEphemeralVolume("test-volume").Pod,
cachePVCs: []*v1.PersistentVolumeClaim{conflictingGenericPVC},
pvs: []*v1.PersistentVolume{pvBoundGeneric},
ephemeral: true,
shouldFail: true,
},
"generic-ephemeral,disabled": {
pod: makePod("test-pod").
withNamespace("testns").
withNodeName("node1").
withGenericEphemeralVolume("test-volume").Pod,
cachePVCs: []*v1.PersistentVolumeClaim{correctGenericPVC},
pvs: []*v1.PersistentVolume{pvBoundGeneric},
ephemeral: false,
shouldFail: true,
},
}
@@ -986,8 +970,6 @@ func TestFindPodVolumesWithoutProvisioning(t *testing.T) {
}
run := func(t *testing.T, scenario scenarioType, csiStorageCapacity bool, csiDriver *storagev1.CSIDriver) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.GenericEphemeralVolume, scenario.ephemeral)()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

36
pkg/scheduler/framework/plugins/volumebinding/volume_binding.go
View File

@@ -25,9 +25,9 @@ import (
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
corelisters "k8s.io/client-go/listers/core/v1"
"k8s.io/component-helpers/storage/ephemeral"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/apis/config/validation"
@@ -69,11 +69,10 @@ func (d *stateData) Clone() framework.StateData {
// In the Filter phase, pod binding cache is created for the pod and used in
// Reserve and PreBind phases.
type VolumeBinding struct {
Binder SchedulerVolumeBinder
PVCLister corelisters.PersistentVolumeClaimLister
GenericEphemeralVolumeFeatureEnabled bool
scorer volumeCapacityScorer
fts feature.Features
Binder SchedulerVolumeBinder
PVCLister corelisters.PersistentVolumeClaimLister
scorer volumeCapacityScorer
fts feature.Features
}
var _ framework.PreFilterPlugin = &VolumeBinding{}
@@ -127,13 +126,13 @@ func (pl *VolumeBinding) podHasPVCs(pod *v1.Pod) (bool, error) {
hasPVC := false
for _, vol := range pod.Spec.Volumes {
var pvcName string
ephemeral := false
isEphemeral := false
switch {
case vol.PersistentVolumeClaim != nil:
pvcName = vol.PersistentVolumeClaim.ClaimName
case vol.Ephemeral != nil && pl.GenericEphemeralVolumeFeatureEnabled:
pvcName = pod.Name + "-" + vol.Name
ephemeral = true
case vol.Ephemeral != nil:
pvcName = ephemeral.VolumeClaimName(pod, &vol)
isEphemeral = true
default:
// Volume is not using a PVC, ignore
continue
@@ -144,7 +143,7 @@ func (pl *VolumeBinding) podHasPVCs(pod *v1.Pod) (bool, error) {
// The error usually has already enough context ("persistentvolumeclaim "myclaim" not found"),
// but we can do better for generic ephemeral inline volumes where that situation
// is normal directly after creating a pod.
if ephemeral && apierrors.IsNotFound(err) {
if isEphemeral && apierrors.IsNotFound(err) {
err = fmt.Errorf("waiting for ephemeral volume controller to create the persistentvolumeclaim %q", pvcName)
}
return hasPVC, err
@@ -158,8 +157,10 @@ func (pl *VolumeBinding) podHasPVCs(pod *v1.Pod) (bool, error) {
return hasPVC, fmt.Errorf("persistentvolumeclaim %q is being deleted", pvc.Name)
}
if ephemeral && !metav1.IsControlledBy(pvc, pod) {
return hasPVC, fmt.Errorf("persistentvolumeclaim %q was not created for the pod", pvc.Name)
if isEphemeral {
if err := ephemeral.VolumeIsForPod(pod, pvc); err != nil {
return hasPVC, err
}
}
}
return hasPVC, nil
@@ -400,10 +401,9 @@ func New(plArgs runtime.Object, fh framework.Handle, fts feature.Features) (fram
scorer = buildScorerFunction(shape)
}
return &VolumeBinding{
Binder: binder,
PVCLister: pvcInformer.Lister(),
GenericEphemeralVolumeFeatureEnabled: fts.EnableGenericEphemeralVolume,
scorer: scorer,
fts: fts,
Binder: binder,
PVCLister: pvcInformer.Lister(),
scorer: scorer,
fts: fts,
}, nil
}