github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 19:56:01 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #50231 from crassirostris/sd-logging-e2e-soak-fix

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 50091, 50231, 50238, 50236, 50243)

Fix Stackdriver Logging soak tests issues

It mostly addressed occasional panics and too verbose logs about ingestion

Fixes https://github.com/kubernetes/kubernetes/issues/50237

/cc @piosz @x13n
This commit is contained in:
Kubernetes Submit Queue 2017-08-07 07:14:54 -07:00 committed by GitHub
commit 5cbb73d616
3 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
test/e2e/instrumentation/logging/stackdrvier/soak.go
View File

@ -84,8 +84,9 @@ var _ = instrumentation.SIGDescribe("Cluster level logging implemented by Stackd
for runIdx := 0; runIdx < podRunCount; runIdx++ { for runIdx := 0; runIdx < podRunCount; runIdx++ {
// Starting one pod on each node. // Starting one pod on each node.
for _, pod := range podsByRun[runIdx] { for _, pod := range podsByRun[runIdx] {
err := pod.Start(f) if err := pod.Start(f); err != nil {
framework.Logf("Failed to start pod: %v", err) framework.Logf("Failed to start pod: %v", err)
}
} }
<-t.C <-t.C
} }

7
test/e2e/instrumentation/logging/utils/logging_agent.go
View File

@ -65,7 +65,12 @@ func EnsureLoggingAgentRestartsCount(f *framework.Framework, appName string, max
maxRestartCount := 0 maxRestartCount := 0
for _, pod := range agentPods.Items { for _, pod := range agentPods.Items {
restartCount := int(pod.Status.ContainerStatuses[0].RestartCount) contStatuses := pod.Status.ContainerStatuses
if len(contStatuses) == 0 {
framework.Logf("There are no container statuses for pod %s", pod.Name)
continue
}
restartCount := int(contStatuses[0].RestartCount)
maxRestartCount = integer.IntMax(maxRestartCount, restartCount) maxRestartCount = integer.IntMax(maxRestartCount, restartCount)
framework.Logf("Logging agent %s on node %s was restarted %d times", framework.Logf("Logging agent %s on node %s was restarted %d times",

14
test/e2e/instrumentation/logging/utils/wait.go
View File

@ -150,9 +150,6 @@ func getFullIngestionPred(podsMap map[string]FiniteLoggingPod) NumberedIngestion
return func(name string, occ map[int]bool) (bool, error) { return func(name string, occ map[int]bool) (bool, error) {
p := podsMap[name] p := podsMap[name]
ok := len(occ) == p.ExpectedLineCount() ok := len(occ) == p.ExpectedLineCount()
if !ok {
framework.Logf("Pod %s is still missing %d lines", name, p.ExpectedLineCount()-len(occ))
}
return ok, nil return ok, nil
} }
} }
@ -160,24 +157,27 @@ func getFullIngestionPred(podsMap map[string]FiniteLoggingPod) NumberedIngestion
func getFullIngestionTimeout(podsMap map[string]FiniteLoggingPod, slack float64) NumberedTimeoutFun { func getFullIngestionTimeout(podsMap map[string]FiniteLoggingPod, slack float64) NumberedTimeoutFun {
return func(names []string, occs map[string]map[int]bool) error { return func(names []string, occs map[string]map[int]bool) error {
totalGot, totalWant := 0, 0 totalGot, totalWant := 0, 0
podsWithLosses := []string{} lossMsgs := []string{}
for _, name := range names { for _, name := range names {
got := len(occs[name]) got := len(occs[name])
want := podsMap[name].ExpectedLineCount() want := podsMap[name].ExpectedLineCount()
if got != want { if got != want {
podsWithLosses = append(podsWithLosses, name) lossMsg := fmt.Sprintf("%s: %d lines", name, want-got)
lossMsgs = append(lossMsgs, lossMsg)
} }
totalGot += got totalGot += got
totalWant += want totalWant += want
} }
if len(podsWithLosses) > 0 { if len(lossMsgs) > 0 {
framework.Logf("Still missing logs from: %s", strings.Join(podsWithLosses, ", ")) framework.Logf("Still missing logs from:\n%s", strings.Join(lossMsgs, "\n"))
} }
lostFrac := 1 - float64(totalGot)/float64(totalWant) lostFrac := 1 - float64(totalGot)/float64(totalWant)
if lostFrac > slack { if lostFrac > slack {
return fmt.Errorf("still missing %.2f%% of logs, only %.2f%% is tolerable", return fmt.Errorf("still missing %.2f%% of logs, only %.2f%% is tolerable",
lostFrac*100, slack*100) lostFrac*100, slack*100)
} }
framework.Logf("Missing %.2f%% of logs, which is lower than the threshold %.2f%%",
lostFrac*100, slack*100)
return nil return nil
} }
} }