github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-27 05:27:21 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #96244 from yue9944882/apf-filter-test-failure

Browse Source 
Fixes APF filter test failure due to racy read&write
This commit is contained in:
Kubernetes Prow Robot 2020-11-09 13:44:42 -08:00 committed by GitHub
commit d9d626f1e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 21 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
staging/src/k8s.io/apiserver/pkg/server/filters/priority-and-fairness_test.go
View File

@ -18,7 +18,6 @@ package filters
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
@ -178,7 +177,7 @@ func TestApfRejectRequest(t *testing.T) {
t.Error(err) t.Error(err)
} }
checkForExpectedMetricsWithRetry(t, []string{ checkForExpectedMetrics(t, []string{
"apiserver_request_terminations_total", "apiserver_request_terminations_total",
"apiserver_dropped_requests_total", "apiserver_dropped_requests_total",
}) })
@ -203,7 +202,7 @@ func TestApfExemptRequest(t *testing.T) {
t.Error(err) t.Error(err)
} }
checkForExpectedMetricsWithRetry(t, []string{ checkForExpectedMetrics(t, []string{
"apiserver_current_inflight_requests", "apiserver_current_inflight_requests",
"apiserver_flowcontrol_read_vs_write_request_count_watermarks", "apiserver_flowcontrol_read_vs_write_request_count_watermarks",
"apiserver_flowcontrol_read_vs_write_request_count_samples", "apiserver_flowcontrol_read_vs_write_request_count_samples",
@ -229,7 +228,7 @@ func TestApfExecuteRequest(t *testing.T) {
t.Error(err) t.Error(err)
} }
checkForExpectedMetricsWithRetry(t, []string{ checkForExpectedMetrics(t, []string{
"apiserver_current_inflight_requests", "apiserver_current_inflight_requests",
"apiserver_current_inqueue_requests", "apiserver_current_inqueue_requests",
"apiserver_flowcontrol_read_vs_write_request_count_watermarks", "apiserver_flowcontrol_read_vs_write_request_count_watermarks",
@ -246,8 +245,11 @@ func TestApfExecuteMultipleRequests(t *testing.T) {
time.Sleep(time.Millisecond * 50) time.Sleep(time.Millisecond * 50)
concurrentRequests := 5 concurrentRequests := 5
var preStartExecute, postStartExecute, preEnqueue, postEnqueue, preDequeue, postDequeue, finishExecute sync.WaitGroup preStartExecute, postStartExecute := &sync.WaitGroup{}, &sync.WaitGroup{}
for _, wg := range []*sync.WaitGroup{&preStartExecute, &postStartExecute, &preEnqueue, &postEnqueue, &preDequeue, &postDequeue, &finishExecute} { preEnqueue, postEnqueue := &sync.WaitGroup{}, &sync.WaitGroup{}
preDequeue, postDequeue := &sync.WaitGroup{}, &sync.WaitGroup{}
finishExecute := &sync.WaitGroup{}
for _, wg := range []*sync.WaitGroup{preStartExecute, postStartExecute, preEnqueue, postEnqueue, preDequeue, postDequeue, finishExecute} {
wg.Add(concurrentRequests) wg.Add(concurrentRequests)
} }
@ -306,7 +308,7 @@ func TestApfExecuteMultipleRequests(t *testing.T) {
} }
wg.Wait() wg.Wait()
checkForExpectedMetricsWithRetry(t, []string{ checkForExpectedMetrics(t, []string{
"apiserver_current_inflight_requests", "apiserver_current_inflight_requests",
"apiserver_current_inqueue_requests", "apiserver_current_inqueue_requests",
"apiserver_flowcontrol_read_vs_write_request_count_watermarks", "apiserver_flowcontrol_read_vs_write_request_count_watermarks",
@ -324,19 +326,15 @@ func TestApfCancelWaitRequest(t *testing.T) {
t.Error(err) t.Error(err)
} }
checkForExpectedMetricsWithRetry(t, []string{ checkForExpectedMetrics(t, []string{
"apiserver_current_inflight_requests", "apiserver_current_inflight_requests",
"apiserver_request_terminations_total", "apiserver_request_terminations_total",
"apiserver_dropped_requests_total", "apiserver_dropped_requests_total",
}) })
} }
// wait async metrics to be collected // gathers and checks the metrics.
func checkForExpectedMetricsWithRetry(t *testing.T, expectedMetrics []string) { func checkForExpectedMetrics(t *testing.T, expectedMetrics []string) {
maxRetries := 5
var checkErrors []error
for i := 0; i < maxRetries; i++ {
t.Logf("Check for expected metrics with retry %d", i)
metricsFamily, err := legacyregistry.DefaultGatherer.Gather() metricsFamily, err := legacyregistry.DefaultGatherer.Gather()
if err != nil { if err != nil {
t.Fatalf("Failed to gather metrics %v", err) t.Fatalf("Failed to gather metrics %v", err)
@ -347,26 +345,11 @@ func checkForExpectedMetricsWithRetry(t *testing.T, expectedMetrics []string) {
metrics[*mf.Name] = mf metrics[*mf.Name] = mf
} }
checkErrors = checkForExpectedMetrics(expectedMetrics, metrics)
if checkErrors == nil {
return
}
time.Sleep(1 * time.Second)
}
for _, checkError := range checkErrors {
t.Error(checkError)
}
}
func checkForExpectedMetrics(expectedMetrics []string, metrics map[string]interface{}) []error {
var errs []error
for _, metricName := range expectedMetrics { for _, metricName := range expectedMetrics {
if _, ok := metrics[metricName]; !ok { if _, ok := metrics[metricName]; !ok {
if !ok { if !ok {
errs = append(errs, errors.New("Scraped metrics did not include expected metric "+metricName)) t.Errorf("Scraped metrics did not include expected metric %s", metricName)
} }
} }
} }
return errs
} }