github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-04 09:49:50 +00:00
Code Issues Projects Releases Wiki Activity

Loosen requirements of cluster logging e2e tests, make them more stable

Browse Source
This commit is contained in:
Mik Vyatskov 2017-03-17 14:13:12 +01:00
parent 223c721d6e
commit c4660222e6
4 changed files with 46 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
test/e2e/cluster_logging_es.go
View File

@ -45,10 +45,10 @@ var _ = framework.KubeDescribe("Cluster level logging using Elasticsearch [Featu
framework.ExpectNoError(err, "Elasticsearch is not working") framework.ExpectNoError(err, "Elasticsearch is not working")
By("Running synthetic logger") By("Running synthetic logger")
pod := createLoggingPod(f, podName, 100, 1*time.Second) pod := createLoggingPod(f, podName, 10*60, 10*time.Minute)
defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{}) defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{})
err = framework.WaitForPodSuccessInNamespace(f.ClientSet, podName, f.Namespace.Name) err = framework.WaitForPodNameRunningInNamespace(f.ClientSet, podName, f.Namespace.Name)
framework.ExpectNoError(err, fmt.Sprintf("Should've successfully waited for pod %s to succeed", podName)) framework.ExpectNoError(err, fmt.Sprintf("Should've successfully waited for pod %s to be running", podName))
By("Waiting for logs to ingest") By("Waiting for logs to ingest")
config := &loggingTestConfig{ config := &loggingTestConfig{
@ -58,7 +58,6 @@ var _ = framework.KubeDescribe("Cluster level logging using Elasticsearch [Featu
MaxAllowedLostFraction: 0, MaxAllowedLostFraction: 0,
MaxAllowedFluentdRestarts: 0, MaxAllowedFluentdRestarts: 0,
} }
err = waitForLogsIngestion(f, config) framework.ExpectNoError(waitForSomeLogs(f, config), "Failed to ingest logs")
framework.ExpectNoError(err, "Failed to ingest logs")
}) })
}) })

12
test/e2e/cluster_logging_gcl.go
View File

@ -26,8 +26,7 @@ import (
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
) )
// TODO(crassirostris): Remove Flaky once test is stable var _ = framework.KubeDescribe("Cluster level logging using GCL", func() {
var _ = framework.KubeDescribe("Cluster level logging using GCL [Flaky]", func() {
f := framework.NewDefaultFramework("gcl-logging") f := framework.NewDefaultFramework("gcl-logging")
BeforeEach(func() { BeforeEach(func() {
@ -44,10 +43,10 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Flaky]", func()
framework.ExpectNoError(err, "GCL is not working") framework.ExpectNoError(err, "GCL is not working")
By("Running synthetic logger") By("Running synthetic logger")
pod := createLoggingPod(f, podName, 100, 1*time.Second) pod := createLoggingPod(f, podName, 10*60, 10*time.Minute)
defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{}) defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{})
err = framework.WaitForPodSuccessInNamespace(f.ClientSet, podName, f.Namespace.Name) err = framework.WaitForPodNameRunningInNamespace(f.ClientSet, podName, f.Namespace.Name)
framework.ExpectNoError(err, fmt.Sprintf("Should've successfully waited for pod %s to succeed", podName)) framework.ExpectNoError(err, fmt.Sprintf("Should've successfully waited for pod %s to be running", podName))
By("Waiting for logs to ingest") By("Waiting for logs to ingest")
config := &loggingTestConfig{ config := &loggingTestConfig{
@ -57,7 +56,6 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Flaky]", func()
MaxAllowedLostFraction: 0, MaxAllowedLostFraction: 0,
MaxAllowedFluentdRestarts: 0, MaxAllowedFluentdRestarts: 0,
} }
err = waitForLogsIngestion(f, config) framework.ExpectNoError(waitForSomeLogs(f, config), "Failed to ingest logs")
framework.ExpectNoError(err, "Failed to ingest logs")
}) })
}) })

4
test/e2e/cluster_logging_gcl_load.go
View File

@ -67,7 +67,7 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Slow] [Flaky]",
MaxAllowedLostFraction: loadTestMaxAllowedLostFraction, MaxAllowedLostFraction: loadTestMaxAllowedLostFraction,
MaxAllowedFluentdRestarts: loadTestMaxAllowedFluentdRestarts, MaxAllowedFluentdRestarts: loadTestMaxAllowedFluentdRestarts,
} }
err = waitForLogsIngestion(f, config) err = waitForFullLogsIngestion(f, config)
if err != nil { if err != nil {
framework.Failf("Failed to ingest logs: %v", err) framework.Failf("Failed to ingest logs: %v", err)
} else { } else {
@ -113,7 +113,7 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Slow] [Flaky]",
MaxAllowedLostFraction: loadTestMaxAllowedLostFraction, MaxAllowedLostFraction: loadTestMaxAllowedLostFraction,
MaxAllowedFluentdRestarts: loadTestMaxAllowedFluentdRestarts, MaxAllowedFluentdRestarts: loadTestMaxAllowedFluentdRestarts,
} }
err = waitForLogsIngestion(f, config) err = waitForFullLogsIngestion(f, config)
if err != nil { if err != nil {
framework.Failf("Failed to ingest logs: %v", err) framework.Failf("Failed to ingest logs: %v", err)
} else { } else {

36
test/e2e/cluster_logging_utils.go
View File

@ -141,7 +141,41 @@ func createLogsGeneratorPod(f *framework.Framework, podName string, linesCount i
}) })
} }
func waitForLogsIngestion(f *framework.Framework, config *loggingTestConfig) error { func waitForSomeLogs(f *framework.Framework, config *loggingTestConfig) error {
podHasIngestedLogs := make([]bool, len(config.Pods))
podWithIngestedLogsCount := 0
for start := time.Now(); podWithIngestedLogsCount < len(config.Pods) && time.Since(start) < config.IngestionTimeout; time.Sleep(ingestionRetryDelay) {
for podIdx, pod := range config.Pods {
if podHasIngestedLogs[podIdx] {
continue
}
entries := config.LogsProvider.ReadEntries(pod)
if len(entries) == 0 {
framework.Logf("No log entries from pod %s", pod.Name)
continue
}
for _, entry := range entries {
if _, ok := entry.getLogEntryNumber(); ok {
framework.Logf("Found some log entries from pod %s", pod.Name)
podHasIngestedLogs[podIdx] = true
podWithIngestedLogsCount++
break
}
}
}
}
if podWithIngestedLogsCount < len(config.Pods) {
return fmt.Errorf("some logs were ingested for %d pods out of %d", podWithIngestedLogsCount, len(config.Pods))
}
return nil
}
func waitForFullLogsIngestion(f *framework.Framework, config *loggingTestConfig) error {
expectedLinesNumber := 0 expectedLinesNumber := 0
for _, pod := range config.Pods { for _, pod := range config.Pods {
expectedLinesNumber += pod.ExpectedLinesNumber expectedLinesNumber += pod.ExpectedLinesNumber