github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-29 22:46:12 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #38244 from MrHohn/e2e-reboot-drop

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 36419, 38330, 37718, 38244, 38375)

Guarantees drop packets commands succeed in reboot test

Fixes the main case in #33405 and #36230.
Previous attempted fix in #38057.

During the reboot test, the iptables command that was supposed to take the node offline failed to exec. 
Turned out the xtables lock was holding by other processes led to this failure. Logs as below:
```
I1202 20:00:29.686] Dec  2 20:00:29.685: INFO: ssh jenkins@146.148.111.167:22: stdout:
"+ sleep 10
+ sudo iptables -I INPUT 1 -s 127.0.0.1 -j ACCEPT
Another app is currently holding the xtables lock. Perhaps you want to use the -w option?"
I1202 20:00:29.686] Dec  2 20:00:29.685: INFO: ssh jenkins@146.148.111.167:22: stderr:    ""
I1202 20:00:29.686] Dec  2 20:00:29.685: INFO: ssh jenkins@146.148.111.167:22: exit code: 0
```

This reboot test won't pass if any one of these iptables commands fails. This PR put "reboot" commands into while loops to guarantee it retries until succeed.

`sudo iptables -t filter -nL` is removed since it is clear now that the `FILTER` rules won't be clobbered.

(Tests passed on local cluster.)

@bprashanth
This commit is contained in:
Kubernetes Submit Queue 2016-12-08 17:13:58 -08:00 committed by GitHub
commit 81dd16a4ae
1 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
test/e2e/reboot.go
View File

@ -18,6 +18,7 @@ package e2e
import (
"fmt"
"strings"
"sync"
"time"
@ -116,9 +117,7 @@ var _ = framework.KubeDescribe("Reboot [Disruptive] [Feature:Reboot]", func() {
// We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping inbound packets.
// We still accept packages send from localhost to prevent monit from restarting kubelet.
tmpLogPath := "/tmp/drop-inbound.log"
testReboot(f.ClientSet, fmt.Sprintf("nohup sh -c 'set -x && sleep 10 && sudo iptables -I INPUT 1 -s 127.0.0.1 -j ACCEPT"+
" && sudo iptables -I INPUT 2 -j DROP && sudo iptables -t filter -nL INPUT && date && sleep 120 && sudo iptables -t filter -nL INPUT"+
" && sudo iptables -D INPUT -j DROP && sudo iptables -D INPUT -s 127.0.0.1 -j ACCEPT' >%v 2>&1 &", tmpLogPath), catLogHook(tmpLogPath))
testReboot(f.ClientSet, dropPacketsScript("INPUT", tmpLogPath), catLogHook(tmpLogPath))
})
It("each node by dropping all outbound packets for a while and ensure they function afterwards", func() {
@ -126,9 +125,7 @@ var _ = framework.KubeDescribe("Reboot [Disruptive] [Feature:Reboot]", func() {
// We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping outbound packets.
// We still accept packages send to localhost to prevent monit from restarting kubelet.
tmpLogPath := "/tmp/drop-outbound.log"
testReboot(f.ClientSet, fmt.Sprintf("nohup sh -c 'set -x && sleep 10 && sudo iptables -I OUTPUT 1 -s 127.0.0.1 -j ACCEPT"+
" && sudo iptables -I OUTPUT 2 -j DROP && sudo iptables -t filter -nL OUTPUT && date && sleep 120 && sudo iptables -t filter -nL OUTPUT"+
" && sudo iptables -D OUTPUT -j DROP && sudo iptables -D OUTPUT -s 127.0.0.1 -j ACCEPT' >%v 2>&1 &", tmpLogPath), catLogHook(tmpLogPath))
testReboot(f.ClientSet, dropPacketsScript("OUTPUT", tmpLogPath), catLogHook(tmpLogPath))
})
})
@ -303,3 +300,18 @@ func catLogHook(logPath string) terminationHook {
}
}
func dropPacketsScript(chainName, logPath string) string {
return strings.Replace(fmt.Sprintf(`
nohup sh -c '
set -x
sleep 10
while true; do sudo iptables -I ${CHAIN} 1 -s 127.0.0.1 -j ACCEPT && break; done
while true; do sudo iptables -I ${CHAIN} 2 -j DROP && break; done
date
sleep 120
while true; do sudo iptables -D ${CHAIN} -j DROP && break; done
while true; do sudo iptables -D ${CHAIN} -s 127.0.0.1 -j ACCEPT && break; done
' >%v 2>&1 &
`, logPath), "${CHAIN}", chainName, -1)
}