diff --git a/test/integration/scheduler_perf/config/performance-config.yaml b/test/integration/scheduler_perf/config/performance-config.yaml index 20d79dcaeb4..86032d82c3f 100644 --- a/test/integration/scheduler_perf/config/performance-config.yaml +++ b/test/integration/scheduler_perf/config/performance-config.yaml @@ -27,15 +27,18 @@ countParam: $initNodes uniqueNodeLabelStrategy: labelKey: kubernetes.io/hostname + - opcode: createNamespaces + prefix: sched + count: 2 - opcode: createPods countParam: $initPods podTemplatePath: config/pod-with-pod-anti-affinity.yaml - namespace: sched-setup + namespace: sched-0 - opcode: createPods countParam: $measurePods podTemplatePath: config/pod-with-pod-anti-affinity.yaml collectMetrics: true - namespace: sched-test + namespace: sched-1 workloads: - name: 500Nodes params: @@ -173,14 +176,17 @@ labelNodePrepareStrategy: labelKey: "topology.kubernetes.io/zone" labelValues: ["zone1"] + - opcode: createNamespaces + prefix: sched + count: 2 - opcode: createPods countParam: $initPods podTemplatePath: config/pod-with-pod-affinity.yaml - namespace: sched-setup + namespace: sched-0 - opcode: createPods countParam: $measurePods podTemplatePath: config/pod-with-pod-affinity.yaml - namespace: sched-test + namespace: sched-1 collectMetrics: true workloads: - name: 500Nodes @@ -200,14 +206,17 @@ countParam: $initNodes uniqueNodeLabelStrategy: labelKey: kubernetes.io/hostname + - opcode: createNamespaces + prefix: sched + count: 2 - opcode: createPods countParam: $initPods podTemplatePath: config/pod-with-preferred-pod-affinity.yaml - namespace: sched-setup + namespace: sched-0 - opcode: createPods countParam: $measurePods podTemplatePath: config/pod-with-preferred-pod-affinity.yaml - namespace: sched-test + namespace: sched-1 collectMetrics: true workloads: - name: 500Nodes @@ -227,14 +236,17 @@ countParam: $initNodes uniqueNodeLabelStrategy: labelKey: kubernetes.io/hostname + - opcode: createNamespaces + prefix: sched + count: 2 - opcode: createPods countParam: $initPods podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml - namespace: sched-setup + namespace: sched-0 - opcode: createPods countParam: $measurePods podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml - namespace: sched-test + namespace: sched-1 collectMetrics: true workloads: - name: 500Nodes @@ -337,26 +349,29 @@ labelNodePrepareStrategy: labelKey: "topology.kubernetes.io/zone" labelValues: ["zone1"] + - opcode: createNamespaces + prefix: sched + count: 1 - opcode: createPods countParam: $initPods podTemplatePath: config/pod-default.yaml - namespace: sched-setup + namespace: sched-0 - opcode: createPods countParam: $initPods podTemplatePath: config/pod-with-pod-affinity.yaml - namespace: sched-setup + namespace: sched-0 - opcode: createPods countParam: $initPods podTemplatePath: config/pod-with-pod-anti-affinity.yaml - namespace: sched-setup + namespace: sched-0 - opcode: createPods countParam: $initPods podTemplatePath: config/pod-with-preferred-pod-affinity.yaml - namespace: sched-setup + namespace: sched-0 - opcode: createPods countParam: $initPods podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml - namespace: sched-setup + namespace: sched-0 - opcode: createPods countParam: $measurePods podTemplatePath: config/pod-default.yaml @@ -478,7 +493,7 @@ - name: SchedulingRequiredPodAntiAffinityWithNSSelector featureGates: - PodAffinityNamespaceSelector: true + PodAffinityNamespaceSelector: true workloadTemplate: - opcode: createNodes countParam: $initNodes @@ -511,10 +526,10 @@ initPodsPerNamespace: 40 initNamespaces: 100 measurePods: 1000 - + - name: SchedulingPreferredAntiAffinityWithNSSelector featureGates: - PodAffinityNamespaceSelector: true + PodAffinityNamespaceSelector: true workloadTemplate: - opcode: createNodes countParam: $initNodes @@ -550,7 +565,7 @@ - name: SchedulingRequiredPodAffinityWithNSSelector featureGates: - PodAffinityNamespaceSelector: true + PodAffinityNamespaceSelector: true workloadTemplate: - opcode: createNodes countParam: $initNodes @@ -584,10 +599,10 @@ initPodsPerNamespace: 50 initNamespaces: 100 measurePods: 1000 - + - name: SchedulingPreferredAffinityWithNSSelector featureGates: - PodAffinityNamespaceSelector: true + PodAffinityNamespaceSelector: true workloadTemplate: - opcode: createNodes countParam: $initNodes @@ -620,4 +635,4 @@ initPodsPerNamespace: 50 initNamespaces: 100 measurePods: 1000 - + diff --git a/test/integration/scheduler_perf/config/pod-with-pod-affinity.yaml b/test/integration/scheduler_perf/config/pod-with-pod-affinity.yaml index a2c969e2e51..401f1830d97 100644 --- a/test/integration/scheduler_perf/config/pod-with-pod-affinity.yaml +++ b/test/integration/scheduler_perf/config/pod-with-pod-affinity.yaml @@ -12,7 +12,7 @@ spec: matchLabels: color: blue topologyKey: topology.kubernetes.io/zone - namespaces: ["sched-test", "sched-setup"] + namespaces: ["sched-1", "sched-0"] containers: - image: k8s.gcr.io/pause:3.6 name: pause diff --git a/test/integration/scheduler_perf/config/pod-with-pod-anti-affinity.yaml b/test/integration/scheduler_perf/config/pod-with-pod-anti-affinity.yaml index 0b46e9e4d55..632d784c533 100644 --- a/test/integration/scheduler_perf/config/pod-with-pod-anti-affinity.yaml +++ b/test/integration/scheduler_perf/config/pod-with-pod-anti-affinity.yaml @@ -12,7 +12,7 @@ spec: matchLabels: color: green topologyKey: kubernetes.io/hostname - namespaces: ["sched-test", "sched-setup"] + namespaces: ["sched-1", "sched-0"] containers: - image: k8s.gcr.io/pause:3.6 name: pause diff --git a/test/integration/scheduler_perf/config/pod-with-preferred-pod-affinity.yaml b/test/integration/scheduler_perf/config/pod-with-preferred-pod-affinity.yaml index d5640d0558d..33a86373042 100644 --- a/test/integration/scheduler_perf/config/pod-with-preferred-pod-affinity.yaml +++ b/test/integration/scheduler_perf/config/pod-with-preferred-pod-affinity.yaml @@ -13,7 +13,7 @@ spec: matchLabels: color: red topologyKey: kubernetes.io/hostname - namespaces: ["sched-test", "sched-setup"] + namespaces: ["sched-1", "sched-0"] weight: 1 containers: - image: k8s.gcr.io/pause:3.6 diff --git a/test/integration/scheduler_perf/config/pod-with-preferred-pod-anti-affinity.yaml b/test/integration/scheduler_perf/config/pod-with-preferred-pod-anti-affinity.yaml index c3757bbd6fb..40ef4489a08 100644 --- a/test/integration/scheduler_perf/config/pod-with-preferred-pod-anti-affinity.yaml +++ b/test/integration/scheduler_perf/config/pod-with-preferred-pod-anti-affinity.yaml @@ -13,7 +13,7 @@ spec: matchLabels: color: yellow topologyKey: kubernetes.io/hostname - namespaces: ["sched-test", "sched-setup"] + namespaces: ["sched-1", "sched-0"] weight: 1 containers: - image: k8s.gcr.io/pause:3.6 diff --git a/test/integration/scheduler_perf/scheduler_perf_test.go b/test/integration/scheduler_perf/scheduler_perf_test.go index fbb7fb0134f..aafb45644f8 100644 --- a/test/integration/scheduler_perf/scheduler_perf_test.go +++ b/test/integration/scheduler_perf/scheduler_perf_test.go @@ -600,8 +600,17 @@ func runWorkload(b *testing.B, tc *testCase, w *workload) []DataItem { var mu sync.Mutex var dataItems []DataItem - numPodsScheduledPerNamespace := make(map[string]int) nextNodeIndex := 0 + // numPodsScheduledPerNamespace has all namespaces created in workload and the number of pods they (will) have. + // All namespaces listed in numPodsScheduledPerNamespace will be cleaned up. + numPodsScheduledPerNamespace := make(map[string]int) + b.Cleanup(func() { + for namespace, _ := range numPodsScheduledPerNamespace { + if err := client.CoreV1().Namespaces().Delete(context.Background(), namespace, metav1.DeleteOptions{}); err != nil { + b.Errorf("Deleting Namespace in numPodsScheduledPerNamespace: %v", err) + } + } + }) for opIndex, op := range unrollWorkloadTemplate(b, tc.WorkloadTemplate, w) { realOp, err := op.realOp.patchParams(w) @@ -636,27 +645,29 @@ func runWorkload(b *testing.B, tc *testCase, w *workload) []DataItem { nsPreparer.cleanup() b.Fatalf("op %d: %v", opIndex, err) } - b.Cleanup(func() { - nsPreparer.cleanup() - }) + for _, n := range nsPreparer.namespaces() { + if _, ok := numPodsScheduledPerNamespace[n]; ok { + // this namespace has been already created. + continue + } + numPodsScheduledPerNamespace[n] = 0 + } case *createPodsOp: var namespace string + // define Pod's namespace automatically, and create that namespace. + namespace = fmt.Sprintf("namespace-%d", opIndex) if concreteOp.Namespace != nil { namespace = *concreteOp.Namespace - } else { - // define Pod's namespace automatically, and create that namespace. - namespace = fmt.Sprintf("namespace-%d", opIndex) + } + if _, ok := numPodsScheduledPerNamespace[namespace]; !ok { + // The namespace has not created yet. + // So, creat that and register it to numPodsScheduledPerNamespace. _, err := client.CoreV1().Namespaces().Create(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { + if err != nil { b.Fatalf("failed to create namespace for Pod: %v", namespace) } - b.Cleanup(func() { - err := client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) - if err != nil { - b.Errorf("failed to delete namespace %v", namespace) - } - }) + numPodsScheduledPerNamespace[namespace] = 0 } var collectors []testDataCollector var collectorCtx context.Context @@ -1104,6 +1115,15 @@ func newNamespacePreparer(cno *createNamespacesOp, clientset clientset.Interface }, nil } +// namespaces returns namespace names have been (or will be) created by this namespacePreparer +func (p *namespacePreparer) namespaces() []string { + namespaces := make([]string, p.count) + for i := 0; i < p.count; i++ { + namespaces[i] = fmt.Sprintf("%s-%d", p.prefix, i) + } + return namespaces +} + // prepare creates the namespaces. func (p *namespacePreparer) prepare() error { base := &v1.Namespace{} @@ -1115,7 +1135,7 @@ func (p *namespacePreparer) prepare() error { n := base.DeepCopy() n.Name = fmt.Sprintf("%s-%d", p.prefix, i) if err := testutils.RetryWithExponentialBackOff(func() (bool, error) { - _, err := p.client.CoreV1().Namespaces().Create(context.TODO(), n, metav1.CreateOptions{}) + _, err := p.client.CoreV1().Namespaces().Create(context.Background(), n, metav1.CreateOptions{}) return err == nil || apierrors.IsAlreadyExists(err), nil }); err != nil { return err @@ -1129,7 +1149,7 @@ func (p *namespacePreparer) cleanup() error { var errRet error for i := 0; i < p.count; i++ { n := fmt.Sprintf("%s-%d", p.prefix, i) - if err := p.client.CoreV1().Namespaces().Delete(context.TODO(), n, metav1.DeleteOptions{}); err != nil { + if err := p.client.CoreV1().Namespaces().Delete(context.Background(), n, metav1.DeleteOptions{}); err != nil { klog.Errorf("Deleting Namespace: %v", err) errRet = err } diff --git a/test/integration/util/util.go b/test/integration/util/util.go index fc3b30fbede..1fd9bc11be5 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -120,6 +120,7 @@ func StartFakePVController(clientSet clientset.Interface) ShutdownFunc { pvInformer := informerFactory.Core().V1().PersistentVolumes() syncPV := func(obj *v1.PersistentVolume) { + ctx := context.Background() if obj.Spec.ClaimRef != nil { claimRef := obj.Spec.ClaimRef pvc, err := clientSet.CoreV1().PersistentVolumeClaims(claimRef.Namespace).Get(ctx, claimRef.Name, metav1.GetOptions{}) diff --git a/test/utils/create_resources.go b/test/utils/create_resources.go index fd8ff2feb98..71a5e1ff79e 100644 --- a/test/utils/create_resources.go +++ b/test/utils/create_resources.go @@ -59,6 +59,9 @@ func CreatePodWithRetries(c clientset.Interface, namespace string, obj *v1.Pod) } createFunc := func() (bool, error) { _, err := c.CoreV1().Pods(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + if isGenerateNameConflict(obj.ObjectMeta, err) { + return false, nil + } if err == nil || apierrors.IsAlreadyExists(err) { return true, nil } @@ -73,6 +76,9 @@ func CreateRCWithRetries(c clientset.Interface, namespace string, obj *v1.Replic } createFunc := func() (bool, error) { _, err := c.CoreV1().ReplicationControllers(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + if isGenerateNameConflict(obj.ObjectMeta, err) { + return false, nil + } if err == nil || apierrors.IsAlreadyExists(err) { return true, nil } @@ -87,6 +93,9 @@ func CreateReplicaSetWithRetries(c clientset.Interface, namespace string, obj *a } createFunc := func() (bool, error) { _, err := c.AppsV1().ReplicaSets(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + if isGenerateNameConflict(obj.ObjectMeta, err) { + return false, nil + } if err == nil || apierrors.IsAlreadyExists(err) { return true, nil } @@ -101,6 +110,9 @@ func CreateDeploymentWithRetries(c clientset.Interface, namespace string, obj *a } createFunc := func() (bool, error) { _, err := c.AppsV1().Deployments(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + if isGenerateNameConflict(obj.ObjectMeta, err) { + return false, nil + } if err == nil || apierrors.IsAlreadyExists(err) { return true, nil } @@ -115,6 +127,9 @@ func CreateDaemonSetWithRetries(c clientset.Interface, namespace string, obj *ap } createFunc := func() (bool, error) { _, err := c.AppsV1().DaemonSets(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + if isGenerateNameConflict(obj.ObjectMeta, err) { + return false, nil + } if err == nil || apierrors.IsAlreadyExists(err) { return true, nil } @@ -129,6 +144,9 @@ func CreateJobWithRetries(c clientset.Interface, namespace string, obj *batch.Jo } createFunc := func() (bool, error) { _, err := c.BatchV1().Jobs(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + if isGenerateNameConflict(obj.ObjectMeta, err) { + return false, nil + } if err == nil || apierrors.IsAlreadyExists(err) { return true, nil } @@ -143,6 +161,9 @@ func CreateSecretWithRetries(c clientset.Interface, namespace string, obj *v1.Se } createFunc := func() (bool, error) { _, err := c.CoreV1().Secrets(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + if isGenerateNameConflict(obj.ObjectMeta, err) { + return false, nil + } if err == nil || apierrors.IsAlreadyExists(err) { return true, nil } @@ -157,6 +178,9 @@ func CreateConfigMapWithRetries(c clientset.Interface, namespace string, obj *v1 } createFunc := func() (bool, error) { _, err := c.CoreV1().ConfigMaps(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + if isGenerateNameConflict(obj.ObjectMeta, err) { + return false, nil + } if err == nil || apierrors.IsAlreadyExists(err) { return true, nil } @@ -171,6 +195,9 @@ func CreateServiceWithRetries(c clientset.Interface, namespace string, obj *v1.S } createFunc := func() (bool, error) { _, err := c.CoreV1().Services(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + if isGenerateNameConflict(obj.ObjectMeta, err) { + return false, nil + } if err == nil || apierrors.IsAlreadyExists(err) { return true, nil } @@ -185,6 +212,9 @@ func CreateStorageClassWithRetries(c clientset.Interface, obj *storage.StorageCl } createFunc := func() (bool, error) { _, err := c.StorageV1().StorageClasses().Create(context.TODO(), obj, metav1.CreateOptions{}) + if isGenerateNameConflict(obj.ObjectMeta, err) { + return false, nil + } if err == nil || apierrors.IsAlreadyExists(err) { return true, nil } @@ -199,6 +229,9 @@ func CreateResourceQuotaWithRetries(c clientset.Interface, namespace string, obj } createFunc := func() (bool, error) { _, err := c.CoreV1().ResourceQuotas(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + if isGenerateNameConflict(obj.ObjectMeta, err) { + return false, nil + } if err == nil || apierrors.IsAlreadyExists(err) { return true, nil } @@ -213,6 +246,9 @@ func CreatePersistentVolumeWithRetries(c clientset.Interface, obj *v1.Persistent } createFunc := func() (bool, error) { _, err := c.CoreV1().PersistentVolumes().Create(context.TODO(), obj, metav1.CreateOptions{}) + if isGenerateNameConflict(obj.ObjectMeta, err) { + return false, nil + } if err == nil || apierrors.IsAlreadyExists(err) { return true, nil } @@ -227,6 +263,9 @@ func CreatePersistentVolumeClaimWithRetries(c clientset.Interface, namespace str } createFunc := func() (bool, error) { _, err := c.CoreV1().PersistentVolumeClaims(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + if isGenerateNameConflict(obj.ObjectMeta, err) { + return false, nil + } if err == nil || apierrors.IsAlreadyExists(err) { return true, nil } @@ -234,3 +273,11 @@ func CreatePersistentVolumeClaimWithRetries(c clientset.Interface, namespace str } return RetryWithExponentialBackOff(createFunc) } + +// isGenerateNameConflict returns whether the error is generateName conflict or not. +func isGenerateNameConflict(meta metav1.ObjectMeta, err error) bool { + if apierrors.IsAlreadyExists(err) && meta.Name == "" { + return true + } + return false +}