From 501a7678b3964bf8eec5e6a21011667098cf57d7 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 14 Feb 2023 10:02:47 +0100 Subject: [PATCH] test/utils: avoid data race during parallel create The client-go Create call writes into the object that it gets passed. Each call therefore needs its own copy when invoked in parallel. Seen in go test -v -timeout=0 -bench=.*/SchedulingBasic/5000Nodes -race ./test/integration/scheduler_perf WARNING: DATA RACE Read at 0x00c003fa5b00 by goroutine 45227: k8s.io/apimachinery/pkg/apis/meta/v1.(*TypeMeta).GroupVersionKind() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/apis/meta/v1/meta.go:126 +0x84 k8s.io/apimachinery/pkg/runtime.WithVersionEncoder.Encode() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/runtime/helper.go:231 +0x176 k8s.io/apimachinery/pkg/runtime.(*WithVersionEncoder).Encode() :1 +0xfb k8s.io/apimachinery/pkg/runtime.Encode() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/runtime/codec.go:50 +0xb3 k8s.io/client-go/rest.(*Request).Body() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/client-go/rest/request.go:469 +0x884 k8s.io/client-go/kubernetes/typed/core/v1.(*pods).Create() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/client-go/kubernetes/typed/core/v1/pod.go:126 +0x264 k8s.io/kubernetes/test/utils.CreatePodWithRetries.func1() /nvme/gopath/src/k8s.io/kubernetes/test/utils/create_resources.go:61 +0x111 k8s.io/apimachinery/pkg/util/wait.ConditionFunc.WithContext.func1() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:222 +0x30 k8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:262 +0x7b k8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:255 +0x5c k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:431 +0x67 k8s.io/kubernetes/test/utils.RetryWithExponentialBackOff() /nvme/gopath/src/k8s.io/kubernetes/test/utils/create_resources.go:53 +0x1be k8s.io/kubernetes/test/utils.CreatePodWithRetries() /nvme/gopath/src/k8s.io/kubernetes/test/utils/create_resources.go:70 +0x1bf k8s.io/kubernetes/test/utils.makeCreatePod() /nvme/gopath/src/k8s.io/kubernetes/test/utils/runners.go:1339 +0x68 k8s.io/kubernetes/test/utils.CreatePod.func1() /nvme/gopath/src/k8s.io/kubernetes/test/utils/runners.go:1349 +0xab k8s.io/client-go/util/workqueue.ParallelizeUntil.func1() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/client-go/util/workqueue/parallelizer.go:90 +0x1c1 Previous write at 0x00c003fa5b00 by goroutine 45250: k8s.io/apimachinery/pkg/apis/meta/v1.(*TypeMeta).SetGroupVersionKind() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/apis/meta/v1/meta.go:121 +0x1cc k8s.io/apimachinery/pkg/runtime.WithVersionEncoder.Encode() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/runtime/helper.go:241 +0x408 k8s.io/apimachinery/pkg/runtime.(*WithVersionEncoder).Encode() :1 +0xfb k8s.io/apimachinery/pkg/runtime.Encode() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/runtime/codec.go:50 +0xb3 k8s.io/client-go/rest.(*Request).Body() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/client-go/rest/request.go:469 +0x884 k8s.io/client-go/kubernetes/typed/core/v1.(*pods).Create() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/client-go/kubernetes/typed/core/v1/pod.go:126 +0x264 k8s.io/kubernetes/test/utils.CreatePodWithRetries.func1() /nvme/gopath/src/k8s.io/kubernetes/test/utils/create_resources.go:61 +0x111 k8s.io/apimachinery/pkg/util/wait.ConditionFunc.WithContext.func1() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:222 +0x30 k8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:262 +0x7b k8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:255 +0x5c k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:431 +0x67 k8s.io/kubernetes/test/utils.RetryWithExponentialBackOff() /nvme/gopath/src/k8s.io/kubernetes/test/utils/create_resources.go:53 +0x1be k8s.io/kubernetes/test/utils.CreatePodWithRetries() /nvme/gopath/src/k8s.io/kubernetes/test/utils/create_resources.go:70 +0x1bf k8s.io/kubernetes/test/utils.makeCreatePod() /nvme/gopath/src/k8s.io/kubernetes/test/utils/runners.go:1339 +0x68 k8s.io/kubernetes/test/utils.CreatePod.func1() /nvme/gopath/src/k8s.io/kubernetes/test/utils/runners.go:1349 +0xab k8s.io/client-go/util/workqueue.ParallelizeUntil.func1() /nvme/gopath/src/k8s.io/kubernetes/vendor/k8s.io/client-go/util/workqueue/parallelizer.go:90 +0x1c1 --- test/utils/runners.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/utils/runners.go b/test/utils/runners.go index 2c3d3015ab6..4abbe4fd4f3 100644 --- a/test/utils/runners.go +++ b/test/utils/runners.go @@ -1346,7 +1346,10 @@ func CreatePod(ctx context.Context, client clientset.Interface, namespace string var createError error lock := sync.Mutex{} createPodFunc := func(i int) { - if err := makeCreatePod(client, namespace, podTemplate); err != nil { + // client-go writes into the object that is passed to Create, + // causing a data race unless we create a new copy for each + // parallel call. + if err := makeCreatePod(client, namespace, podTemplate.DeepCopy()); err != nil { lock.Lock() defer lock.Unlock() createError = err