From bfe41927ba77fff2313a30e20fce55df5cec3b34 Mon Sep 17 00:00:00 2001 From: "Timothy St. Clair" Date: Thu, 19 Jan 2017 15:11:20 -0600 Subject: [PATCH] Cleaup Affinity post conversion from annotations to fields --- cmd/kubeadm/app/master/addons.go | 37 +++++++++++- cmd/kubeadm/app/master/apiclient.go | 32 ----------- cmd/kubeadm/app/master/discovery.go | 19 ++++++- pkg/api/helpers.go | 4 -- pkg/api/v1/helpers.go | 18 ------ pkg/api/v1/helpers_test.go | 56 ------------------- .../src/k8s.io/client-go/pkg/api/helpers.go | 4 -- .../k8s.io/client-go/pkg/api/v1/helpers.go | 18 ------ test/integration/scheduler_perf/BUILD | 1 - .../scheduler_perf/scheduler_test.go | 48 ++++++++-------- 10 files changed, 78 insertions(+), 159 deletions(-) diff --git a/cmd/kubeadm/app/master/addons.go b/cmd/kubeadm/app/master/addons.go index 9d682fe0792..62c00619c03 100644 --- a/cmd/kubeadm/app/master/addons.go +++ b/cmd/kubeadm/app/master/addons.go @@ -20,6 +20,7 @@ import ( "fmt" "net" "path" + "runtime" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -81,6 +82,23 @@ func createKubeProxyPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec { }, }, }, + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "beta.kubernetes.io/arch", + Operator: v1.NodeSelectorOpIn, + Values: []string{runtime.GOARCH}, + }, + }, + }, + }, + }, + }, + }, } } @@ -240,6 +258,23 @@ func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec { }, }, DNSPolicy: v1.DNSDefault, + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "beta.kubernetes.io/arch", + Operator: v1.NodeSelectorOpIn, + Values: []string{runtime.GOARCH}, + }, + }, + }, + }, + }, + }, + }, } } @@ -266,7 +301,6 @@ func createKubeDNSServiceSpec(cfg *kubeadmapi.MasterConfiguration) (*v1.ServiceS func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error { kubeProxyDaemonSet := NewDaemonSet(kubeProxy, createKubeProxyPodSpec(cfg)) SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta) - SetNodeAffinity(&kubeProxyDaemonSet.Spec.Template.ObjectMeta, NativeArchitectureNodeAffinity()) if _, err := client.Extensions().DaemonSets(metav1.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil { return fmt.Errorf("failed creating essential kube-proxy addon [%v]", err) @@ -276,7 +310,6 @@ func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client *clientse kubeDNSDeployment := NewDeployment(KubeDNS, 1, createKubeDNSPodSpec(cfg)) SetMasterTaintTolerations(&kubeDNSDeployment.Spec.Template.ObjectMeta) - SetNodeAffinity(&kubeDNSDeployment.Spec.Template.ObjectMeta, NativeArchitectureNodeAffinity()) kubeDNSServiceAccount := &v1.ServiceAccount{} kubeDNSServiceAccount.ObjectMeta.Name = KubeDNS if _, err := client.ServiceAccounts(metav1.NamespaceSystem).Create(kubeDNSServiceAccount); err != nil { diff --git a/cmd/kubeadm/app/master/apiclient.go b/cmd/kubeadm/app/master/apiclient.go index d1331ee6e6c..d00d76c89c2 100644 --- a/cmd/kubeadm/app/master/apiclient.go +++ b/cmd/kubeadm/app/master/apiclient.go @@ -19,7 +19,6 @@ package master import ( "encoding/json" "fmt" - "runtime" "time" apierrs "k8s.io/apimachinery/pkg/api/errors" @@ -229,37 +228,6 @@ func SetMasterTaintTolerations(meta *metav1.ObjectMeta) { meta.Annotations[v1.TolerationsAnnotationKey] = string(tolerationsAnnotation) } -// SetNodeAffinity is a basic helper to set meta.Annotations[v1.AffinityAnnotationKey] for one or more v1.NodeSelectorRequirement(s) -func SetNodeAffinity(meta *metav1.ObjectMeta, expr ...v1.NodeSelectorRequirement) { - nodeAffinity := &v1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{{MatchExpressions: expr}}, - }, - } - affinityAnnotation, _ := json.Marshal(v1.Affinity{NodeAffinity: nodeAffinity}) - if meta.Annotations == nil { - meta.Annotations = map[string]string{} - } - meta.Annotations[v1.AffinityAnnotationKey] = string(affinityAnnotation) -} - -// MasterNodeAffinity returns v1.NodeSelectorRequirement to be used with SetNodeAffinity to set affinity to master node -func MasterNodeAffinity() v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{ - Key: metav1.NodeLabelKubeadmAlphaRole, - Operator: v1.NodeSelectorOpIn, - Values: []string{metav1.NodeLabelRoleMaster}, - } -} - -// NativeArchitectureNodeAffinity returns v1.NodeSelectorRequirement to be used with SetNodeAffinity to nodes with CPU architecture -// the same as master node -func NativeArchitectureNodeAffinity() v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{ - Key: "beta.kubernetes.io/arch", Operator: v1.NodeSelectorOpIn, Values: []string{runtime.GOARCH}, - } -} - func createDummyDeployment(client *clientset.Clientset) { fmt.Println("[apiclient] Creating a test deployment") dummyDeployment := NewDeployment("dummy", 1, v1.PodSpec{ diff --git a/cmd/kubeadm/app/master/discovery.go b/cmd/kubeadm/app/master/discovery.go index cc4d2b44341..205a3d5f65c 100644 --- a/cmd/kubeadm/app/master/discovery.go +++ b/cmd/kubeadm/app/master/discovery.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "path" + "runtime" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -103,6 +104,23 @@ func newKubeDiscoveryPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec { Secret: &v1.SecretVolumeSource{SecretName: kubeDiscoverySecretName}, }}, }, + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "beta.kubernetes.io/arch", + Operator: v1.NodeSelectorOpIn, + Values: []string{runtime.GOARCH}, + }, + }, + }, + }, + }, + }, + }, } } @@ -117,7 +135,6 @@ func newKubeDiscovery(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certific } SetMasterTaintTolerations(&kd.Deployment.Spec.Template.ObjectMeta) - SetNodeAffinity(&kd.Deployment.Spec.Template.ObjectMeta, MasterNodeAffinity(), NativeArchitectureNodeAffinity()) return kd } diff --git a/pkg/api/helpers.go b/pkg/api/helpers.go index 7dcf93bac80..3ae3da78fc2 100644 --- a/pkg/api/helpers.go +++ b/pkg/api/helpers.go @@ -429,10 +429,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S } const ( - // AffinityAnnotationKey represents the key of affinity data (json serialized) - // in the Annotations of a Pod. - AffinityAnnotationKey string = "scheduler.alpha.kubernetes.io/affinity" - // TolerationsAnnotationKey represents the key of tolerations data (json serialized) // in the Annotations of a Pod. TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations" diff --git a/pkg/api/v1/helpers.go b/pkg/api/v1/helpers.go index f7f5ec0d7cf..97a636a1984 100644 --- a/pkg/api/v1/helpers.go +++ b/pkg/api/v1/helpers.go @@ -233,10 +233,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S } const ( - // AffinityAnnotationKey represents the key of affinity data (json serialized) - // in the Annotations of a Pod. - AffinityAnnotationKey string = "scheduler.alpha.kubernetes.io/affinity" - // TolerationsAnnotationKey represents the key of tolerations data (json serialized) // in the Annotations of a Pod. TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations" @@ -276,20 +272,6 @@ const ( UnsafeSysctlsPodAnnotationKey string = "security.alpha.kubernetes.io/unsafe-sysctls" ) -// GetAffinityFromPod gets the json serialized affinity data from Pod.Annotations -// and converts it to the Affinity type in api. -func GetAffinityFromPodAnnotations(annotations map[string]string) (*Affinity, error) { - if len(annotations) > 0 && annotations[AffinityAnnotationKey] != "" { - var affinity Affinity - err := json.Unmarshal([]byte(annotations[AffinityAnnotationKey]), &affinity) - if err != nil { - return nil, err - } - return &affinity, nil - } - return nil, nil -} - // GetTolerationsFromPodAnnotations gets the json serialized tolerations data from Pod.Annotations // and converts it to the []Toleration type in api. func GetTolerationsFromPodAnnotations(annotations map[string]string) ([]Toleration, error) { diff --git a/pkg/api/v1/helpers_test.go b/pkg/api/v1/helpers_test.go index 72ffabe83eb..c4716de38e0 100644 --- a/pkg/api/v1/helpers_test.go +++ b/pkg/api/v1/helpers_test.go @@ -179,62 +179,6 @@ func TestNodeSelectorRequirementsAsSelector(t *testing.T) { } } -func TestGetAffinityFromPod(t *testing.T) { - testCases := []struct { - pod *Pod - expectErr bool - }{ - { - pod: &Pod{}, - expectErr: false, - }, - { - pod: &Pod{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "foo", - "operator": "In", - "values": ["value1", "value2"] - }] - }] - }}}`, - }, - }, - }, - expectErr: false, - }, - { - pod: &Pod{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "foo", - `, - }, - }, - }, - expectErr: true, - }, - } - - for i, tc := range testCases { - _, err := GetAffinityFromPodAnnotations(tc.pod.Annotations) - if err == nil && tc.expectErr { - t.Errorf("[%v]expected error but got none.", i) - } - if err != nil && !tc.expectErr { - t.Errorf("[%v]did not expect error but got: %v", i, err) - } - } -} - func TestTaintToString(t *testing.T) { testCases := []struct { taint *Taint diff --git a/staging/src/k8s.io/client-go/pkg/api/helpers.go b/staging/src/k8s.io/client-go/pkg/api/helpers.go index d0c2dd0e1c5..32a48e3af4b 100644 --- a/staging/src/k8s.io/client-go/pkg/api/helpers.go +++ b/staging/src/k8s.io/client-go/pkg/api/helpers.go @@ -451,10 +451,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S } const ( - // AffinityAnnotationKey represents the key of affinity data (json serialized) - // in the Annotations of a Pod. - AffinityAnnotationKey string = "scheduler.alpha.kubernetes.io/affinity" - // TolerationsAnnotationKey represents the key of tolerations data (json serialized) // in the Annotations of a Pod. TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations" diff --git a/staging/src/k8s.io/client-go/pkg/api/v1/helpers.go b/staging/src/k8s.io/client-go/pkg/api/v1/helpers.go index 7463e68ff74..bd7e72384a0 100644 --- a/staging/src/k8s.io/client-go/pkg/api/v1/helpers.go +++ b/staging/src/k8s.io/client-go/pkg/api/v1/helpers.go @@ -233,10 +233,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S } const ( - // AffinityAnnotationKey represents the key of affinity data (json serialized) - // in the Annotations of a Pod. - AffinityAnnotationKey string = "scheduler.alpha.kubernetes.io/affinity" - // TolerationsAnnotationKey represents the key of tolerations data (json serialized) // in the Annotations of a Pod. TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations" @@ -276,20 +272,6 @@ const ( UnsafeSysctlsPodAnnotationKey string = "security.alpha.kubernetes.io/unsafe-sysctls" ) -// GetAffinityFromPod gets the json serialized affinity data from Pod.Annotations -// and converts it to the Affinity type in api. -func GetAffinityFromPodAnnotations(annotations map[string]string) (*Affinity, error) { - if len(annotations) > 0 && annotations[AffinityAnnotationKey] != "" { - var affinity Affinity - err := json.Unmarshal([]byte(annotations[AffinityAnnotationKey]), &affinity) - if err != nil { - return nil, err - } - return &affinity, nil - } - return nil, nil -} - // GetTolerationsFromPodAnnotations gets the json serialized tolerations data from Pod.Annotations // and converts it to the []Toleration type in api. func GetTolerationsFromPodAnnotations(annotations map[string]string) ([]Toleration, error) { diff --git a/test/integration/scheduler_perf/BUILD b/test/integration/scheduler_perf/BUILD index 263b5506fda..b6c9d6de7e9 100644 --- a/test/integration/scheduler_perf/BUILD +++ b/test/integration/scheduler_perf/BUILD @@ -41,7 +41,6 @@ go_test( "//test/integration/framework:go_default_library", "//test/utils:go_default_library", "//vendor:github.com/golang/glog", - "//vendor:github.com/renstrom/dedent", "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", ], ) diff --git a/test/integration/scheduler_perf/scheduler_test.go b/test/integration/scheduler_perf/scheduler_test.go index 46436d64925..190aaffe7f8 100644 --- a/test/integration/scheduler_perf/scheduler_test.go +++ b/test/integration/scheduler_perf/scheduler_test.go @@ -28,7 +28,6 @@ import ( testutils "k8s.io/kubernetes/test/utils" "github.com/golang/glog" - "github.com/renstrom/dedent" "k8s.io/kubernetes/plugin/pkg/scheduler" ) @@ -79,31 +78,34 @@ func TestSchedule100Node3KNodeAffinityPods(t *testing.T) { "scheduler-perf-", ) - affinityTemplate := dedent.Dedent(` - { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "` + nodeAffinityKey + `", - "operator": "In", - "values": ["%v"] - }] - }] - } - } - }`) - podCreatorConfig := testutils.NewTestPodCreatorConfig() for i := 0; i < numGroups; i++ { - podCreatorConfig.AddStrategy("sched-perf-node-affinity", config.numPods/numGroups, - testutils.NewCustomCreatePodStrategy(&v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "sched-perf-node-affinity-pod-", - Annotations: map[string]string{v1.AffinityAnnotationKey: fmt.Sprintf(affinityTemplate, i)}, + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "sched-perf-node-affinity-pod-", + }, + Spec: testutils.MakePodSpec(), + } + pod.Spec.Affinity = &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: nodeAffinityKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(i)}, + }, + }, + }, + }, }, - Spec: testutils.MakePodSpec(), - }), + }, + } + + podCreatorConfig.AddStrategy("sched-perf-node-affinity", config.numPods/numGroups, + testutils.NewCustomCreatePodStrategy(pod), ) } config.podCreator = testutils.NewTestPodCreator(config.schedulerSupportFunctions.GetClient(), podCreatorConfig)