github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-31 08:36:16 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #43294 from crassirostris/cluster-logging-test-simplifying

Browse Source 
Automatic merge from submit-queue

Loosen requirements of cluster logging e2e tests, make them more stable

There should be an e2e test for cloud logging in the main test suite, because this is the important part of functionality and it can be broken by different components.

However, existing cluster logging e2e tests were too strict for the current solution, which may loose some log entries, which results in flakes. There's no way to fix this problem in 1.6, so this PR makes basic cluster logging e2e tests less strict.
This commit is contained in:
Kubernetes Submit Queue
2017-03-20 06:04:59 -07:00
committed by GitHub
parent 47320fd3f0 c4660222e6
commit 38055983e0
4 changed files with 46 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
test/e2e/cluster_logging_es.go
View File

@@ -45,10 +45,10 @@ var _ = framework.KubeDescribe("Cluster level logging using Elasticsearch [Featu
framework.ExpectNoError(err, "Elasticsearch is not working")
By("Running synthetic logger")
pod := createLoggingPod(f, podName, 100, 1*time.Second)
pod := createLoggingPod(f, podName, 10*60, 10*time.Minute)
defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{})
err = framework.WaitForPodSuccessInNamespace(f.ClientSet, podName, f.Namespace.Name)
framework.ExpectNoError(err, fmt.Sprintf("Should've successfully waited for pod %s to succeed", podName))
err = framework.WaitForPodNameRunningInNamespace(f.ClientSet, podName, f.Namespace.Name)
framework.ExpectNoError(err, fmt.Sprintf("Should've successfully waited for pod %s to be running", podName))
By("Waiting for logs to ingest")
config := &loggingTestConfig{
@@ -58,7 +58,6 @@ var _ = framework.KubeDescribe("Cluster level logging using Elasticsearch [Featu
MaxAllowedLostFraction: 0,
MaxAllowedFluentdRestarts: 0,
}
err = waitForLogsIngestion(f, config)
framework.ExpectNoError(err, "Failed to ingest logs")
framework.ExpectNoError(waitForSomeLogs(f, config), "Failed to ingest logs")
})
})

12
test/e2e/cluster_logging_gcl.go
View File

@@ -26,8 +26,7 @@ import (
. "github.com/onsi/ginkgo"
)
// TODO(crassirostris): Remove Flaky once test is stable
var _ = framework.KubeDescribe("Cluster level logging using GCL [Flaky]", func() {
var _ = framework.KubeDescribe("Cluster level logging using GCL", func() {
f := framework.NewDefaultFramework("gcl-logging")
BeforeEach(func() {
@@ -44,10 +43,10 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Flaky]", func()
framework.ExpectNoError(err, "GCL is not working")
By("Running synthetic logger")
pod := createLoggingPod(f, podName, 100, 1*time.Second)
pod := createLoggingPod(f, podName, 10*60, 10*time.Minute)
defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{})
err = framework.WaitForPodSuccessInNamespace(f.ClientSet, podName, f.Namespace.Name)
framework.ExpectNoError(err, fmt.Sprintf("Should've successfully waited for pod %s to succeed", podName))
err = framework.WaitForPodNameRunningInNamespace(f.ClientSet, podName, f.Namespace.Name)
framework.ExpectNoError(err, fmt.Sprintf("Should've successfully waited for pod %s to be running", podName))
By("Waiting for logs to ingest")
config := &loggingTestConfig{
@@ -57,7 +56,6 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Flaky]", func()
MaxAllowedLostFraction: 0,
MaxAllowedFluentdRestarts: 0,
}
err = waitForLogsIngestion(f, config)
framework.ExpectNoError(err, "Failed to ingest logs")
framework.ExpectNoError(waitForSomeLogs(f, config), "Failed to ingest logs")
})
})

4
test/e2e/cluster_logging_gcl_load.go
View File

@@ -67,7 +67,7 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Slow] [Flaky]",
MaxAllowedLostFraction: loadTestMaxAllowedLostFraction,
MaxAllowedFluentdRestarts: loadTestMaxAllowedFluentdRestarts,
}
err = waitForLogsIngestion(f, config)
err = waitForFullLogsIngestion(f, config)
if err != nil {
framework.Failf("Failed to ingest logs: %v", err)
} else {
@@ -113,7 +113,7 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Slow] [Flaky]",
MaxAllowedLostFraction: loadTestMaxAllowedLostFraction,
MaxAllowedFluentdRestarts: loadTestMaxAllowedFluentdRestarts,
}
err = waitForLogsIngestion(f, config)
err = waitForFullLogsIngestion(f, config)
if err != nil {
framework.Failf("Failed to ingest logs: %v", err)
} else {

36
test/e2e/cluster_logging_utils.go
View File

@@ -141,7 +141,41 @@ func createLogsGeneratorPod(f *framework.Framework, podName string, linesCount i
})
}
func waitForLogsIngestion(f *framework.Framework, config *loggingTestConfig) error {
func waitForSomeLogs(f *framework.Framework, config *loggingTestConfig) error {
podHasIngestedLogs := make([]bool, len(config.Pods))
podWithIngestedLogsCount := 0
for start := time.Now(); podWithIngestedLogsCount < len(config.Pods) && time.Since(start) < config.IngestionTimeout; time.Sleep(ingestionRetryDelay) {
for podIdx, pod := range config.Pods {
if podHasIngestedLogs[podIdx] {
continue
}
entries := config.LogsProvider.ReadEntries(pod)
if len(entries) == 0 {
framework.Logf("No log entries from pod %s", pod.Name)
continue
}
for _, entry := range entries {
if _, ok := entry.getLogEntryNumber(); ok {
framework.Logf("Found some log entries from pod %s", pod.Name)
podHasIngestedLogs[podIdx] = true
podWithIngestedLogsCount++
break
}
}
}
}
if podWithIngestedLogsCount < len(config.Pods) {
return fmt.Errorf("some logs were ingested for %d pods out of %d", podWithIngestedLogsCount, len(config.Pods))
}
return nil
}
func waitForFullLogsIngestion(f *framework.Framework, config *loggingTestConfig) error {
expectedLinesNumber := 0
for _, pod := range config.Pods {
expectedLinesNumber += pod.ExpectedLinesNumber