mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 19:56:01 +00:00
Merge pull request #111238 from bobbypage/fix-npd-test
test: Only Check NPD logs in standalone mode
This commit is contained in:
commit
e348a76e10
@ -113,77 +113,73 @@ var _ = SIGDescribe("NodeProblemDetector", func() {
|
|||||||
result, err := e2essh.SSH(cmd, host, framework.TestContext.Provider)
|
result, err := e2essh.SSH(cmd, host, framework.TestContext.Provider)
|
||||||
isStandaloneMode[host] = (err == nil && result.Code == 0)
|
isStandaloneMode[host] = (err == nil && result.Code == 0)
|
||||||
|
|
||||||
ginkgo.By(fmt.Sprintf("Check node %q has node-problem-detector process", host))
|
|
||||||
// Using brackets "[n]" is a trick to prevent grep command itself from
|
|
||||||
// showing up, because string text "[n]ode-problem-detector" does not
|
|
||||||
// match regular expression "[n]ode-problem-detector".
|
|
||||||
psCmd := "ps aux | grep [n]ode-problem-detector"
|
|
||||||
result, err = e2essh.SSH(psCmd, host, framework.TestContext.Provider)
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
framework.ExpectEqual(result.Code, 0)
|
|
||||||
gomega.Expect(result.Stdout).To(gomega.ContainSubstring("node-problem-detector"))
|
|
||||||
|
|
||||||
ginkgo.By(fmt.Sprintf("Check node-problem-detector is running fine on node %q", host))
|
|
||||||
journalctlCmd := "sudo journalctl -r -u node-problem-detector"
|
|
||||||
result, err = e2essh.SSH(journalctlCmd, host, framework.TestContext.Provider)
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
framework.ExpectEqual(result.Code, 0)
|
|
||||||
gomega.Expect(result.Stdout).NotTo(gomega.ContainSubstring("node-problem-detector.service: Failed"))
|
|
||||||
|
|
||||||
// Let's assume that node problem detector has started the same time as kubelet
|
|
||||||
// We only will check for the KubeletStart if parsing of date here succeed
|
|
||||||
// This is an optimization to not ssh one more time to get kubelet's start time.
|
|
||||||
// Also we assume specific datetime format to simplify the logic
|
|
||||||
output := result.Stdout
|
|
||||||
|
|
||||||
// searching for the line "Apr 14 04:47:42 gke-cluster-1-default-pool-b1565719-eqht systemd[1]: Started Kubernetes node problem detector." and fallback to
|
|
||||||
idx := strings.Index(output, "Started Kubernetes node problem detector")
|
|
||||||
if idx != -1 {
|
|
||||||
output = output[:idx]
|
|
||||||
idx = strings.LastIndex(output, "\n")
|
|
||||||
|
|
||||||
if idx != -1 {
|
|
||||||
output = output[0:15]
|
|
||||||
}
|
|
||||||
|
|
||||||
st, err := time.Parse("Jan 02 15:04:05", output)
|
|
||||||
st = st.AddDate(time.Now().Year(), 0, 0)
|
|
||||||
|
|
||||||
if err == nil {
|
|
||||||
checkForKubeletStart = time.Since(st) < time.Hour
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// fallback to searching the line:
|
|
||||||
// -- Logs begin at Thu 2022-04-28 17:32:39 UTC, end at Thu 2022-04-28 17:40:05 UTC. --
|
|
||||||
idx := strings.Index(output, ", end at ")
|
|
||||||
|
|
||||||
output = output[:idx]
|
|
||||||
idx = strings.LastIndex(output, "-- Logs begin at ")
|
|
||||||
if idx != -1 {
|
|
||||||
output = output[17:]
|
|
||||||
}
|
|
||||||
|
|
||||||
st, err := time.Parse("Mon 2006-01-02 15:04:05 MST", output)
|
|
||||||
|
|
||||||
if err == nil {
|
|
||||||
checkForKubeletStart = time.Since(st) < time.Hour
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !checkForKubeletStart {
|
|
||||||
ginkgo.By("KubeletStart event will NOT be checked")
|
|
||||||
}
|
|
||||||
|
|
||||||
if isStandaloneMode[host] {
|
if isStandaloneMode[host] {
|
||||||
|
ginkgo.By(fmt.Sprintf("Check node %q has node-problem-detector process", host))
|
||||||
|
// Using brackets "[n]" is a trick to prevent grep command itself from
|
||||||
|
// showing up, because string text "[n]ode-problem-detector" does not
|
||||||
|
// match regular expression "[n]ode-problem-detector".
|
||||||
|
psCmd := "ps aux | grep [n]ode-problem-detector"
|
||||||
|
result, err = e2essh.SSH(psCmd, host, framework.TestContext.Provider)
|
||||||
|
framework.ExpectNoError(err)
|
||||||
|
framework.ExpectEqual(result.Code, 0)
|
||||||
|
gomega.Expect(result.Stdout).To(gomega.ContainSubstring("node-problem-detector"))
|
||||||
|
|
||||||
|
ginkgo.By(fmt.Sprintf("Check node-problem-detector is running fine on node %q", host))
|
||||||
|
journalctlCmd := "sudo journalctl -r -u node-problem-detector"
|
||||||
|
result, err = e2essh.SSH(journalctlCmd, host, framework.TestContext.Provider)
|
||||||
|
framework.ExpectNoError(err)
|
||||||
|
framework.ExpectEqual(result.Code, 0)
|
||||||
|
gomega.Expect(result.Stdout).NotTo(gomega.ContainSubstring("node-problem-detector.service: Failed"))
|
||||||
|
|
||||||
|
// Let's assume that node problem detector has started the same time as kubelet
|
||||||
|
// We only will check for the KubeletStart if parsing of date here succeed
|
||||||
|
// This is an optimization to not ssh one more time to get kubelet's start time.
|
||||||
|
// Also we assume specific datetime format to simplify the logic
|
||||||
|
output := result.Stdout
|
||||||
|
|
||||||
|
// searching for the line "Apr 14 04:47:42 gke-cluster-1-default-pool-b1565719-eqht systemd[1]: Started Kubernetes node problem detector." and fallback to
|
||||||
|
idx := strings.Index(output, "Started Kubernetes node problem detector")
|
||||||
|
if idx != -1 {
|
||||||
|
output = output[:idx]
|
||||||
|
idx = strings.LastIndex(output, "\n")
|
||||||
|
|
||||||
|
if idx != -1 {
|
||||||
|
output = output[0:15]
|
||||||
|
}
|
||||||
|
|
||||||
|
st, err := time.Parse("Jan 02 15:04:05", output)
|
||||||
|
st = st.AddDate(time.Now().Year(), 0, 0)
|
||||||
|
|
||||||
|
if err == nil {
|
||||||
|
checkForKubeletStart = time.Since(st) < time.Hour
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// fallback to searching the line:
|
||||||
|
// -- Logs begin at Thu 2022-04-28 17:32:39 UTC, end at Thu 2022-04-28 17:40:05 UTC. --
|
||||||
|
idx := strings.Index(output, ", end at ")
|
||||||
|
|
||||||
|
output = output[:idx]
|
||||||
|
idx = strings.LastIndex(output, "-- Logs begin at ")
|
||||||
|
if idx != -1 {
|
||||||
|
output = output[17:]
|
||||||
|
}
|
||||||
|
|
||||||
|
st, err := time.Parse("Mon 2006-01-02 15:04:05 MST", output)
|
||||||
|
|
||||||
|
if err == nil {
|
||||||
|
checkForKubeletStart = time.Since(st) < time.Hour
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
cpuUsage, uptime := getCPUStat(f, host)
|
cpuUsage, uptime := getCPUStat(f, host)
|
||||||
cpuUsageStats[host] = append(cpuUsageStats[host], cpuUsage)
|
cpuUsageStats[host] = append(cpuUsageStats[host], cpuUsage)
|
||||||
uptimeStats[host] = append(uptimeStats[host], uptime)
|
uptimeStats[host] = append(uptimeStats[host], uptime)
|
||||||
}
|
|
||||||
|
|
||||||
|
}
|
||||||
ginkgo.By(fmt.Sprintf("Inject log to trigger DockerHung on node %q", host))
|
ginkgo.By(fmt.Sprintf("Inject log to trigger DockerHung on node %q", host))
|
||||||
log := "INFO: task docker:12345 blocked for more than 120 seconds."
|
log := "INFO: task docker:12345 blocked for more than 120 seconds."
|
||||||
injectLogCmd := "sudo sh -c \"echo 'kernel: " + log + "' >> /dev/kmsg\""
|
injectLogCmd := "sudo sh -c \"echo 'kernel: " + log + "' >> /dev/kmsg\""
|
||||||
_, err = e2essh.SSH(injectLogCmd, host, framework.TestContext.Provider)
|
result, err = e2essh.SSH(injectLogCmd, host, framework.TestContext.Provider)
|
||||||
framework.ExpectNoError(err)
|
framework.ExpectNoError(err)
|
||||||
framework.ExpectEqual(result.Code, 0)
|
framework.ExpectEqual(result.Code, 0)
|
||||||
}
|
}
|
||||||
@ -211,6 +207,8 @@ var _ = SIGDescribe("NodeProblemDetector", func() {
|
|||||||
gomega.Eventually(func() error {
|
gomega.Eventually(func() error {
|
||||||
return verifyEventExists(f, eventListOptions, "KubeletStart", node.Name)
|
return verifyEventExists(f, eventListOptions, "KubeletStart", node.Name)
|
||||||
}, pollTimeout, pollInterval).Should(gomega.Succeed())
|
}, pollTimeout, pollInterval).Should(gomega.Succeed())
|
||||||
|
} else {
|
||||||
|
ginkgo.By("KubeletStart event will NOT be checked")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user