github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-09-25 20:29:56 +00:00
Code Issues Projects Releases Wiki Activity

kubelet: set both deprecated Beta and GA labels for zone/region topology from the cloud provider

Browse Source 
Signed-off-by: Andrew Sy Kim <kiman@vmware.com>
This commit is contained in:
Andrew Sy Kim
2019-08-14 14:35:42 -04:00
parent 3032c81187
commit 4c194d52da
34 changed files with 1483 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
plugin/pkg/admission/noderestriction/admission_test.go
View File

@@ -163,11 +163,11 @@ func setAllowedUpdateLabels(node *api.Node, value string) *api.Node {
node.Labels["kubernetes.io/hostname"] = value
node.Labels["failure-domain.beta.kubernetes.io/zone"] = value
node.Labels["failure-domain.beta.kubernetes.io/region"] = value
node.Labels["topology.kubernetes.io/zone"] = value
node.Labels["topology.kubernetes.io/region"] = value
node.Labels["beta.kubernetes.io/instance-type"] = value
node.Labels["beta.kubernetes.io/os"] = value
node.Labels["beta.kubernetes.io/arch"] = value
node.Labels["failure-domain.kubernetes.io/zone"] = value
node.Labels["failure-domain.kubernetes.io/region"] = value
node.Labels["kubernetes.io/instance-type"] = value
node.Labels["kubernetes.io/os"] = value
node.Labels["kubernetes.io/arch"] = value

74
plugin/pkg/admission/storage/persistentvolume/label/admission.go
View File

@@ -22,6 +22,7 @@ import (
"errors"
"fmt"
"io"
"reflect"
"sync"
v1 "k8s.io/api/core/v1"
@@ -117,11 +118,13 @@ func (l *persistentVolumeLabel) Admit(ctx context.Context, a admission.Attribute
return admission.NewForbidden(a, err)
}
requirements := make([]api.NodeSelectorRequirement, 0)
if len(volumeLabels) != 0 {
if volume.Labels == nil {
volume.Labels = make(map[string]string)
}
deprecatedTopologyReqs := make([]api.NodeSelectorRequirement, 0)
topologyReqs := make([]api.NodeSelectorRequirement, 0)
for k, v := range volumeLabels {
// We (silently) replace labels if they are provided.
// This should be OK because they are in the kubernetes.io namespace
@@ -129,8 +132,11 @@ func (l *persistentVolumeLabel) Admit(ctx context.Context, a admission.Attribute
volume.Labels[k] = v
// Set NodeSelectorRequirements based on the labels
//
// we currently set both beta (failure-domain.beta.kubernetes.io/zone) and
// GA (topology.kubernetes.io/zone) topology labels for volumes
var values []string
if k == v1.LabelZoneFailureDomain {
if k == v1.LabelZoneFailureDomain || k == v1.LabelZoneFailureDomainStable {
zones, err := volumehelpers.LabelZonesToSet(v)
if err != nil {
return admission.NewForbidden(a, fmt.Errorf("failed to convert label string for Zone: %s to a Set", v))
@@ -140,7 +146,17 @@ func (l *persistentVolumeLabel) Admit(ctx context.Context, a admission.Attribute
} else {
values = []string{v}
}
requirements = append(requirements, api.NodeSelectorRequirement{Key: k, Operator: api.NodeSelectorOpIn, Values: values})
// separate topology requirements based on deprecated vs stable zone/region labels
// all other labels apply to both requirements
if k == v1.LabelZoneFailureDomain || k == v1.LabelZoneRegion {
deprecatedTopologyReqs = append(deprecatedTopologyReqs, api.NodeSelectorRequirement{Key: k, Operator: api.NodeSelectorOpIn, Values: values})
} else if k == v1.LabelZoneFailureDomainStable || k == v1.LabelZoneRegionStable {
topologyReqs = append(topologyReqs, api.NodeSelectorRequirement{Key: k, Operator: api.NodeSelectorOpIn, Values: values})
} else {
deprecatedTopologyReqs = append(deprecatedTopologyReqs, api.NodeSelectorRequirement{Key: k, Operator: api.NodeSelectorOpIn, Values: values})
topologyReqs = append(topologyReqs, api.NodeSelectorRequirement{Key: k, Operator: api.NodeSelectorOpIn, Values: values})
}
}
if volume.Spec.NodeAffinity == nil {
@@ -153,16 +169,44 @@ func (l *persistentVolumeLabel) Admit(ctx context.Context, a admission.Attribute
// Need at least one term pre-allocated whose MatchExpressions can be appended to
volume.Spec.NodeAffinity.Required.NodeSelectorTerms = make([]api.NodeSelectorTerm, 1)
}
if nodeSelectorRequirementKeysExistInNodeSelectorTerms(requirements, volume.Spec.NodeAffinity.Required.NodeSelectorTerms) {
klog.V(4).Infof("NodeSelectorRequirements for cloud labels %v conflict with existing NodeAffinity %v. Skipping addition of NodeSelectorRequirements for cloud labels.",
requirements, volume.Spec.NodeAffinity)
} else {
for _, req := range requirements {
for i := range volume.Spec.NodeAffinity.Required.NodeSelectorTerms {
volume.Spec.NodeAffinity.Required.NodeSelectorTerms[i].MatchExpressions = append(volume.Spec.NodeAffinity.Required.NodeSelectorTerms[i].MatchExpressions, req)
}
deprecatedNodeTerms := volume.DeepCopy().Spec.NodeAffinity.Required.NodeSelectorTerms
stableNodeTerms := volume.DeepCopy().Spec.NodeAffinity.Required.NodeSelectorTerms
// only attempt to rewrite beta/stable topology labels if there are no conflicting labels on the PV at all
if nodeSelectorRequirementKeysExistInNodeSelectorTerms(deprecatedTopologyReqs, volume.Spec.NodeAffinity.Required.NodeSelectorTerms) ||
nodeSelectorRequirementKeysExistInNodeSelectorTerms(topologyReqs, volume.Spec.NodeAffinity.Required.NodeSelectorTerms) {
klog.V(4).Infof("NodeSelectorRequirements for cloud labels %v conflict with existing NodeAffinity %v or %v. Skipping addition of NodeSelectorRequirements for cloud labels.",
deprecatedTopologyReqs, topologyReqs, volume.Spec.NodeAffinity)
return nil
}
for _, req := range deprecatedTopologyReqs {
for i := range deprecatedNodeTerms {
deprecatedNodeTerms[i].MatchExpressions = append(deprecatedNodeTerms[i].MatchExpressions, req)
}
}
for _, req := range topologyReqs {
for i := range stableNodeTerms {
stableNodeTerms[i].MatchExpressions = append(stableNodeTerms[i].MatchExpressions, req)
}
}
// Deprecated and stable node selector terms are the same, i.e. the cloud provider
// didn't specify eithr zone/region labels. In the case set either one without
// expanding the set of selector terms
if reflect.DeepEqual(deprecatedTopologyReqs, topologyReqs) {
volume.Spec.NodeAffinity.Required.NodeSelectorTerms = stableNodeTerms
return nil
}
// for deprecated topology labels, we overwrite existing terms directly with the deprecated topology reqs appended
volume.Spec.NodeAffinity.Required.NodeSelectorTerms = deprecatedNodeTerms
// for new stable topology labels, we expand node selector requirements by copying existing selector terms but using stable topology labels
volume.Spec.NodeAffinity.Required.NodeSelectorTerms = append(volume.Spec.NodeAffinity.Required.NodeSelectorTerms, stableNodeTerms...)
}
return nil
@@ -171,15 +215,17 @@ func (l *persistentVolumeLabel) Admit(ctx context.Context, a admission.Attribute
func (l *persistentVolumeLabel) findVolumeLabels(volume *api.PersistentVolume) (map[string]string, error) {
existingLabels := volume.Labels
// All cloud providers set only these two labels.
// deprecated zone/region labels are still the source of truth
domain, domainOK := existingLabels[v1.LabelZoneFailureDomain]
region, regionOK := existingLabels[v1.LabelZoneRegion]
isDynamicallyProvisioned := metav1.HasAnnotation(volume.ObjectMeta, persistentvolume.AnnDynamicallyProvisioned)
if isDynamicallyProvisioned && domainOK && regionOK {
// PV already has all the labels and we can trust the dynamic provisioning that it provided correct values.
return map[string]string{
v1.LabelZoneFailureDomain: domain,
v1.LabelZoneRegion: region,
v1.LabelZoneFailureDomain: domain,
v1.LabelZoneRegion: region,
v1.LabelZoneFailureDomainStable: domain,
v1.LabelZoneRegionStable: region,
}, nil
}

277
plugin/pkg/admission/storage/persistentvolume/label/admission_test.go
View File

@@ -66,9 +66,10 @@ func Test_PVLAdmission(t *testing.T) {
name: "non-cloud PV ignored",
handler: newPersistentVolumeLabel(),
pvlabeler: mockVolumeLabels(map[string]string{
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
v1.LabelZoneFailureDomainStable: "1__2__3",
}),
preAdmissionPV: &api.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{Name: "noncloud", Namespace: "myns"},
@@ -174,9 +175,10 @@ func Test_PVLAdmission(t *testing.T) {
name: "AWS EBS PV labeled correctly",
handler: newPersistentVolumeLabel(),
pvlabeler: mockVolumeLabels(map[string]string{
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
v1.LabelZoneFailureDomainStable: "1__2__3",
}),
preAdmissionPV: &api.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{Name: "awsebs", Namespace: "myns"},
@@ -193,9 +195,10 @@ func Test_PVLAdmission(t *testing.T) {
Name: "awsebs",
Namespace: "myns",
Labels: map[string]string{
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
v1.LabelZoneFailureDomainStable: "1__2__3",
},
},
Spec: api.PersistentVolumeSpec{
@@ -226,6 +229,25 @@ func Test_PVLAdmission(t *testing.T) {
},
},
},
{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: "a",
Operator: api.NodeSelectorOpIn,
Values: []string{"1"},
},
{
Key: "b",
Operator: api.NodeSelectorOpIn,
Values: []string{"2"},
},
{
Key: v1.LabelZoneFailureDomainStable,
Operator: api.NodeSelectorOpIn,
Values: []string{"1", "2", "3"},
},
},
},
},
},
},
@@ -237,15 +259,19 @@ func Test_PVLAdmission(t *testing.T) {
name: "existing labels from dynamic provisioning are not changed",
handler: newPersistentVolumeLabel(),
pvlabeler: mockVolumeLabels(map[string]string{
v1.LabelZoneFailureDomain: "domain1",
v1.LabelZoneRegion: "region1",
v1.LabelZoneFailureDomain: "domain1",
v1.LabelZoneRegion: "region1",
v1.LabelZoneFailureDomainStable: "domain1",
v1.LabelZoneRegionStable: "region1",
}),
preAdmissionPV: &api.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: "awsebs", Namespace: "myns",
Labels: map[string]string{
v1.LabelZoneFailureDomain: "existingDomain",
v1.LabelZoneRegion: "existingRegion",
v1.LabelZoneFailureDomain: "existingDomain",
v1.LabelZoneRegion: "existingRegion",
v1.LabelZoneFailureDomainStable: "existingDomain",
v1.LabelZoneRegionStable: "existingRegion",
},
Annotations: map[string]string{
persistentvolume.AnnDynamicallyProvisioned: "kubernetes.io/aws-ebs",
@@ -264,8 +290,10 @@ func Test_PVLAdmission(t *testing.T) {
Name: "awsebs",
Namespace: "myns",
Labels: map[string]string{
v1.LabelZoneFailureDomain: "existingDomain",
v1.LabelZoneRegion: "existingRegion",
v1.LabelZoneFailureDomain: "existingDomain",
v1.LabelZoneRegion: "existingRegion",
v1.LabelZoneFailureDomainStable: "existingDomain",
v1.LabelZoneRegionStable: "existingRegion",
},
Annotations: map[string]string{
persistentvolume.AnnDynamicallyProvisioned: "kubernetes.io/aws-ebs",
@@ -294,6 +322,20 @@ func Test_PVLAdmission(t *testing.T) {
},
},
},
{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: v1.LabelZoneRegionStable,
Operator: api.NodeSelectorOpIn,
Values: []string{"existingRegion"},
},
{
Key: v1.LabelZoneFailureDomainStable,
Operator: api.NodeSelectorOpIn,
Values: []string{"existingDomain"},
},
},
},
},
},
},
@@ -305,8 +347,10 @@ func Test_PVLAdmission(t *testing.T) {
name: "existing labels from user are changed",
handler: newPersistentVolumeLabel(),
pvlabeler: mockVolumeLabels(map[string]string{
v1.LabelZoneFailureDomain: "domain1",
v1.LabelZoneRegion: "region1",
v1.LabelZoneFailureDomain: "domain1",
v1.LabelZoneRegion: "region1",
v1.LabelZoneFailureDomainStable: "domain1",
v1.LabelZoneRegionStable: "region1",
}),
preAdmissionPV: &api.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
@@ -329,8 +373,10 @@ func Test_PVLAdmission(t *testing.T) {
Name: "awsebs",
Namespace: "myns",
Labels: map[string]string{
v1.LabelZoneFailureDomain: "domain1",
v1.LabelZoneRegion: "region1",
v1.LabelZoneFailureDomain: "domain1",
v1.LabelZoneRegion: "region1",
v1.LabelZoneFailureDomainStable: "domain1",
v1.LabelZoneRegionStable: "region1",
},
},
Spec: api.PersistentVolumeSpec{
@@ -356,6 +402,20 @@ func Test_PVLAdmission(t *testing.T) {
},
},
},
{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: v1.LabelZoneRegionStable,
Operator: api.NodeSelectorOpIn,
Values: []string{"region1"},
},
{
Key: v1.LabelZoneFailureDomainStable,
Operator: api.NodeSelectorOpIn,
Values: []string{"domain1"},
},
},
},
},
},
},
@@ -367,9 +427,10 @@ func Test_PVLAdmission(t *testing.T) {
name: "GCE PD PV labeled correctly",
handler: newPersistentVolumeLabel(),
pvlabeler: mockVolumeLabels(map[string]string{
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
v1.LabelZoneFailureDomainStable: "1__2__3",
}),
preAdmissionPV: &api.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{Name: "gcepd", Namespace: "myns"},
@@ -386,9 +447,10 @@ func Test_PVLAdmission(t *testing.T) {
Name: "gcepd",
Namespace: "myns",
Labels: map[string]string{
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
v1.LabelZoneFailureDomainStable: "1__2__3",
},
},
Spec: api.PersistentVolumeSpec{
@@ -419,6 +481,25 @@ func Test_PVLAdmission(t *testing.T) {
},
},
},
{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: "a",
Operator: api.NodeSelectorOpIn,
Values: []string{"1"},
},
{
Key: "b",
Operator: api.NodeSelectorOpIn,
Values: []string{"2"},
},
{
Key: v1.LabelZoneFailureDomainStable,
Operator: api.NodeSelectorOpIn,
Values: []string{"1", "2", "3"},
},
},
},
},
},
},
@@ -430,9 +511,10 @@ func Test_PVLAdmission(t *testing.T) {
name: "Azure Disk PV labeled correctly",
handler: newPersistentVolumeLabel(),
pvlabeler: mockVolumeLabels(map[string]string{
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
v1.LabelZoneFailureDomainStable: "1__2__3",
}),
preAdmissionPV: &api.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
@@ -452,9 +534,10 @@ func Test_PVLAdmission(t *testing.T) {
Name: "azurepd",
Namespace: "myns",
Labels: map[string]string{
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
v1.LabelZoneFailureDomainStable: "1__2__3",
},
},
Spec: api.PersistentVolumeSpec{
@@ -485,6 +568,25 @@ func Test_PVLAdmission(t *testing.T) {
},
},
},
{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: "a",
Operator: api.NodeSelectorOpIn,
Values: []string{"1"},
},
{
Key: "b",
Operator: api.NodeSelectorOpIn,
Values: []string{"2"},
},
{
Key: v1.LabelZoneFailureDomainStable,
Operator: api.NodeSelectorOpIn,
Values: []string{"1", "2", "3"},
},
},
},
},
},
},
@@ -496,9 +598,10 @@ func Test_PVLAdmission(t *testing.T) {
name: "Cinder Disk PV labeled correctly",
handler: newPersistentVolumeLabel(),
pvlabeler: mockVolumeLabels(map[string]string{
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
v1.LabelZoneFailureDomainStable: "1__2__3",
}),
preAdmissionPV: &api.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
@@ -518,9 +621,10 @@ func Test_PVLAdmission(t *testing.T) {
Name: "azurepd",
Namespace: "myns",
Labels: map[string]string{
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
v1.LabelZoneFailureDomainStable: "1__2__3",
},
},
Spec: api.PersistentVolumeSpec{
@@ -551,6 +655,25 @@ func Test_PVLAdmission(t *testing.T) {
},
},
},
{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: "a",
Operator: api.NodeSelectorOpIn,
Values: []string{"1"},
},
{
Key: "b",
Operator: api.NodeSelectorOpIn,
Values: []string{"2"},
},
{
Key: v1.LabelZoneFailureDomainStable,
Operator: api.NodeSelectorOpIn,
Values: []string{"1", "2", "3"},
},
},
},
},
},
},
@@ -562,9 +685,10 @@ func Test_PVLAdmission(t *testing.T) {
name: "AWS EBS PV overrides user applied labels",
handler: newPersistentVolumeLabel(),
pvlabeler: mockVolumeLabels(map[string]string{
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
v1.LabelZoneFailureDomainStable: "1__2__3",
}),
preAdmissionPV: &api.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
@@ -587,9 +711,10 @@ func Test_PVLAdmission(t *testing.T) {
Name: "awsebs",
Namespace: "myns",
Labels: map[string]string{
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
v1.LabelZoneFailureDomainStable: "1__2__3",
},
},
Spec: api.PersistentVolumeSpec{
@@ -620,6 +745,25 @@ func Test_PVLAdmission(t *testing.T) {
},
},
},
{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: "a",
Operator: api.NodeSelectorOpIn,
Values: []string{"1"},
},
{
Key: "b",
Operator: api.NodeSelectorOpIn,
Values: []string{"2"},
},
{
Key: v1.LabelZoneFailureDomainStable,
Operator: api.NodeSelectorOpIn,
Values: []string{"1", "2", "3"},
},
},
},
},
},
},
@@ -819,9 +963,10 @@ func Test_PVLAdmission(t *testing.T) {
name: "vSphere PV labeled correctly",
handler: newPersistentVolumeLabel(),
pvlabeler: mockVolumeLabels(map[string]string{
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
v1.LabelZoneFailureDomainStable: "1__2__3",
}),
preAdmissionPV: &api.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
@@ -841,9 +986,10 @@ func Test_PVLAdmission(t *testing.T) {
Name: "vSpherePV",
Namespace: "myns",
Labels: map[string]string{
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
"a": "1",
"b": "2",
v1.LabelZoneFailureDomain: "1__2__3",
v1.LabelZoneFailureDomainStable: "1__2__3",
},
},
Spec: api.PersistentVolumeSpec{
@@ -874,6 +1020,25 @@ func Test_PVLAdmission(t *testing.T) {
},
},
},
{
MatchExpressions: []api.NodeSelectorRequirement{
{
Key: "a",
Operator: api.NodeSelectorOpIn,
Values: []string{"1"},
},
{
Key: "b",
Operator: api.NodeSelectorOpIn,
Values: []string{"2"},
},
{
Key: v1.LabelZoneFailureDomainStable,
Operator: api.NodeSelectorOpIn,
Values: []string{"1", "2", "3"},
},
},
},
},
},
},
@@ -900,6 +1065,8 @@ func Test_PVLAdmission(t *testing.T) {
if !reflect.DeepEqual(testcase.preAdmissionPV, testcase.postAdmissionPV) {
t.Logf("expected PV: %+v", testcase.postAdmissionPV)
t.Logf("actual PV: %+v", testcase.preAdmissionPV)
t.Logf("expected PV node affinity: %+v", testcase.postAdmissionPV.Spec.NodeAffinity.Required)
t.Logf("actual PV node affinity: %+v", testcase.preAdmissionPV.Spec.NodeAffinity.Required)
t.Error("unexpected PV")
}
@@ -927,10 +1094,12 @@ func sortMatchExpressions(pv *api.PersistentVolume) {
return
}
match := pv.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions
sort.Slice(match, func(i, j int) bool {
return match[i].Key < match[j].Key
})
for t := range pv.Spec.NodeAffinity.Required.NodeSelectorTerms {
match := pv.Spec.NodeAffinity.Required.NodeSelectorTerms[t].MatchExpressions
sort.Slice(match, func(i, j int) bool {
return match[i].Key < match[j].Key
})
pv.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions = match
pv.Spec.NodeAffinity.Required.NodeSelectorTerms[t].MatchExpressions = match
}
}