mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-04 01:40:07 +00:00
Log node info on test failures
This commit is contained in:
parent
17c5c0922b
commit
b76fef7d48
@ -102,6 +102,8 @@ func (f *Framework) afterEach() {
|
|||||||
// you may or may not see the killing/deletion/cleanup events.
|
// you may or may not see the killing/deletion/cleanup events.
|
||||||
|
|
||||||
dumpAllPodInfo(f.Client)
|
dumpAllPodInfo(f.Client)
|
||||||
|
|
||||||
|
dumpAllNodeInfo(f.Client)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check whether all nodes are ready after the test.
|
// Check whether all nodes are ready after the test.
|
||||||
|
@ -26,7 +26,6 @@ import (
|
|||||||
"k8s.io/kubernetes/pkg/labels"
|
"k8s.io/kubernetes/pkg/labels"
|
||||||
|
|
||||||
. "github.com/onsi/ginkgo"
|
. "github.com/onsi/ginkgo"
|
||||||
. "github.com/onsi/gomega"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -44,13 +43,9 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var _ = Describe("Reboot", func() {
|
var _ = Describe("Reboot", func() {
|
||||||
var c *client.Client
|
f := NewFramework("reboot")
|
||||||
|
|
||||||
BeforeEach(func() {
|
BeforeEach(func() {
|
||||||
var err error
|
|
||||||
c, err = loadClient()
|
|
||||||
Expect(err).NotTo(HaveOccurred())
|
|
||||||
|
|
||||||
// These tests requires SSH to nodes, so the provider check should be identical to there
|
// These tests requires SSH to nodes, so the provider check should be identical to there
|
||||||
// (the limiting factor is the implementation of util.go's getSigner(...)).
|
// (the limiting factor is the implementation of util.go's getSigner(...)).
|
||||||
|
|
||||||
@ -61,32 +56,32 @@ var _ = Describe("Reboot", func() {
|
|||||||
It("each node by ordering clean reboot and ensure they function upon restart", func() {
|
It("each node by ordering clean reboot and ensure they function upon restart", func() {
|
||||||
// clean shutdown and restart
|
// clean shutdown and restart
|
||||||
// We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is rebooted.
|
// We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is rebooted.
|
||||||
testReboot(c, "nohup sh -c 'sleep 10 && sudo reboot' >/dev/null 2>&1 &")
|
testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo reboot' >/dev/null 2>&1 &")
|
||||||
})
|
})
|
||||||
|
|
||||||
It("each node by ordering unclean reboot and ensure they function upon restart", func() {
|
It("each node by ordering unclean reboot and ensure they function upon restart", func() {
|
||||||
// unclean shutdown and restart
|
// unclean shutdown and restart
|
||||||
// We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is shutdown.
|
// We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is shutdown.
|
||||||
testReboot(c, "nohup sh -c 'sleep 10 && echo b | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &")
|
testReboot(f.Client, "nohup sh -c 'sleep 10 && echo b | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &")
|
||||||
})
|
})
|
||||||
|
|
||||||
It("each node by triggering kernel panic and ensure they function upon restart", func() {
|
It("each node by triggering kernel panic and ensure they function upon restart", func() {
|
||||||
// kernel panic
|
// kernel panic
|
||||||
// We sleep 10 seconds to give some time for ssh command to cleanly finish before kernel panic is triggered.
|
// We sleep 10 seconds to give some time for ssh command to cleanly finish before kernel panic is triggered.
|
||||||
testReboot(c, "nohup sh -c 'sleep 10 && echo c | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &")
|
testReboot(f.Client, "nohup sh -c 'sleep 10 && echo c | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &")
|
||||||
})
|
})
|
||||||
|
|
||||||
It("each node by switching off the network interface and ensure they function upon switch on", func() {
|
It("each node by switching off the network interface and ensure they function upon switch on", func() {
|
||||||
// switch the network interface off for a while to simulate a network outage
|
// switch the network interface off for a while to simulate a network outage
|
||||||
// We sleep 10 seconds to give some time for ssh command to cleanly finish before network is down.
|
// We sleep 10 seconds to give some time for ssh command to cleanly finish before network is down.
|
||||||
testReboot(c, "nohup sh -c 'sleep 10 && sudo ifdown eth0 && sleep 120 && sudo ifup eth0' >/dev/null 2>&1 &")
|
testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo ifdown eth0 && sleep 120 && sudo ifup eth0' >/dev/null 2>&1 &")
|
||||||
})
|
})
|
||||||
|
|
||||||
It("each node by dropping all inbound packets for a while and ensure they function afterwards", func() {
|
It("each node by dropping all inbound packets for a while and ensure they function afterwards", func() {
|
||||||
// tell the firewall to drop all inbound packets for a while
|
// tell the firewall to drop all inbound packets for a while
|
||||||
// We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping inbound packets.
|
// We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping inbound packets.
|
||||||
// We still accept packages send from localhost to prevent monit from restarting kubelet.
|
// We still accept packages send from localhost to prevent monit from restarting kubelet.
|
||||||
testReboot(c, "nohup sh -c 'sleep 10 && sudo iptables -I INPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I INPUT 2 -j DROP && "+
|
testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo iptables -I INPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I INPUT 2 -j DROP && "+
|
||||||
" sleep 120 && sudo iptables -D INPUT -j DROP && sudo iptables -D INPUT -s 127.0.0.1 -j ACCEPT' >/dev/null 2>&1 &")
|
" sleep 120 && sudo iptables -D INPUT -j DROP && sudo iptables -D INPUT -s 127.0.0.1 -j ACCEPT' >/dev/null 2>&1 &")
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -94,7 +89,7 @@ var _ = Describe("Reboot", func() {
|
|||||||
// tell the firewall to drop all outbound packets for a while
|
// tell the firewall to drop all outbound packets for a while
|
||||||
// We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping outbound packets.
|
// We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping outbound packets.
|
||||||
// We still accept packages send to localhost to prevent monit from restarting kubelet.
|
// We still accept packages send to localhost to prevent monit from restarting kubelet.
|
||||||
testReboot(c, "nohup sh -c 'sleep 10 && sudo iptables -I OUTPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I OUTPUT 2 -j DROP && "+
|
testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo iptables -I OUTPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I OUTPUT 2 -j DROP && "+
|
||||||
" sleep 120 && sudo iptables -D OUTPUT -j DROP && sudo iptables -D OUTPUT -s 127.0.0.1 -j ACCEPT' >/dev/null 2>&1 &")
|
" sleep 120 && sudo iptables -D OUTPUT -j DROP && sudo iptables -D OUTPUT -s 127.0.0.1 -j ACCEPT' >/dev/null 2>&1 &")
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
@ -1477,8 +1477,28 @@ func dumpAllPodInfo(c *client.Client) {
|
|||||||
logPodStates(pods.Items)
|
logPodStates(pods.Items)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func dumpAllNodeInfo(c *client.Client) {
|
||||||
|
nodes, err := c.Nodes().List(labels.Everything(), fields.Everything())
|
||||||
|
if err != nil {
|
||||||
|
Logf("unable to fetch node list: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
names := make([]string, len(nodes.Items))
|
||||||
|
for ix := range nodes.Items {
|
||||||
|
names[ix] = nodes.Items[ix].Name
|
||||||
|
}
|
||||||
|
dumpNodeDebugInfo(c, names)
|
||||||
|
}
|
||||||
|
|
||||||
func dumpNodeDebugInfo(c *client.Client, nodeNames []string) {
|
func dumpNodeDebugInfo(c *client.Client, nodeNames []string) {
|
||||||
for _, n := range nodeNames {
|
for _, n := range nodeNames {
|
||||||
|
Logf("\nLogging node info for node %v", n)
|
||||||
|
node, err := c.Nodes().Get(n)
|
||||||
|
if err != nil {
|
||||||
|
Logf("Error getting node info %v", err)
|
||||||
|
}
|
||||||
|
Logf("Node Info: %v", node)
|
||||||
|
|
||||||
Logf("\nLogging kubelet events for node %v", n)
|
Logf("\nLogging kubelet events for node %v", n)
|
||||||
for _, e := range getNodeEvents(c, n) {
|
for _, e := range getNodeEvents(c, n) {
|
||||||
Logf("source %v message %v reason %v first ts %v last ts %v, involved obj %+v",
|
Logf("source %v message %v reason %v first ts %v last ts %v, involved obj %+v",
|
||||||
|
Loading…
Reference in New Issue
Block a user