diff --git a/test/integration/framework/etcd.go b/test/integration/framework/etcd.go index 50e27b170d1..741477cc6b7 100644 --- a/test/integration/framework/etcd.go +++ b/test/integration/framework/etcd.go @@ -32,6 +32,7 @@ import ( "google.golang.org/grpc/grpclog" "k8s.io/klog/v2" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/pkg/util/env" ) @@ -192,22 +193,25 @@ func EtcdMain(tests func() int) { result := tests() stop() // Don't defer this. See os.Exit documentation. - // Log the number of goroutines leaked. - // TODO: we should work on reducing this as much as possible. - var dg int - for i := 0; i < 10; i++ { + checkNumberOfGoroutines := func() (bool, error) { // Leave some room for goroutines we can not get rid of // like k8s.io/klog/v2.(*loggingT).flushDaemon() - // TODO: adjust this number based on a more exhaustive analysis. - if dg = runtime.NumGoroutine() - before; dg <= 4 { - os.Exit(result) + // TODO(#108483): Reduce this number once we address the + // couple remaining issues. + if dg := runtime.NumGoroutine() - before; dg <= 10 { + return true, nil } // Allow goroutines to schedule and die off. runtime.Gosched() - time.Sleep(100 * time.Millisecond) + return false, nil + } + + // It generally takes visibly less than 1s to finish all goroutines. + // But we keep the limit higher to account for cpu-starved environments. + if err := wait.Poll(100*time.Millisecond, 5*time.Second, checkNumberOfGoroutines); err != nil { + after := runtime.NumGoroutine() + klog.Fatalf("unexpected number of goroutines: before: %d after %d", before, after) } - after := runtime.NumGoroutine() - klog.Infof("unexpected number of goroutines: before: %d after %d", before, after) os.Exit(result) }