e2e: test that both kube-proxy and kubelet recover after iptables flush

test/e2e/network/networking.go

@@ -19,10 +19,14 @@ package network
 import (
 	"fmt"
 	"net/http"
+	"strings"
 
 	"k8s.io/apimachinery/pkg/util/sets"
+	utilwait "k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/master/ports"
 	"k8s.io/kubernetes/test/e2e/framework"
+	e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
+	e2essh "k8s.io/kubernetes/test/e2e/framework/ssh"
 
 	"github.com/onsi/ginkgo"
 )
@@ -235,4 +239,88 @@ var _ = SIGDescribe("Networking", func() {
 			}
 		})
 	})
+
+	ginkgo.It("should recreate its iptables rules if they are deleted [Disruptive]", func() {
+		framework.SkipUnlessProviderIs(framework.ProvidersWithSSH...)
+		framework.SkipUnlessSSHKeyPresent()
+
+		hosts, err := e2essh.NodeSSHHosts(f.ClientSet)
+		framework.ExpectNoError(err, "failed to find external/internal IPs for every node")
+		if len(hosts) == 0 {
+			framework.Failf("No ssh-able nodes")
+		}
+		host := hosts[0]
+
+		ns := f.Namespace.Name
+		numPods, servicePort := 3, defaultServeHostnameServicePort
+		svc := "iptables-flush-test"
+
+		defer func() {
+			framework.ExpectNoError(e2eservice.StopServeHostnameService(f.ClientSet, ns, svc))
+		}()
+		podNames, svcIP, err := e2eservice.StartServeHostnameService(f.ClientSet, getServeHostnameService(svc), ns, numPods)
+		framework.ExpectNoError(err, "failed to create replication controller with service: %s in the namespace: %s", svc, ns)
+
+		// Ideally we want to reload the system firewall, but we don't necessarily
+		// know how to do that on this system ("firewall-cmd --reload"? "systemctl
+		// restart iptables"?). So instead we just manually delete all "KUBE-"
+		// chains.
+
+		ginkgo.By("dumping iptables rules on a node")
+		result, err := e2essh.SSH("sudo iptables-save", host, framework.TestContext.Provider)
+		if err != nil || result.Code != 0 {
+			e2essh.LogResult(result)
+			framework.Failf("couldn't dump iptable rules: %v", err)
+		}
+
+		// All the commands that delete rules have to come before all the commands
+		// that delete chains, since the chains can't be deleted while there are
+		// still rules referencing them.
+		var deleteRuleCmds, deleteChainCmds []string
+		table := ""
+		for _, line := range strings.Split(result.Stdout, "\n") {
+			if strings.HasPrefix(line, "*") {
+				table = line[1:]
+			} else if table == "" {
+				continue
+			}
+
+			// Delete jumps from non-KUBE chains to KUBE chains
+			if !strings.HasPrefix(line, "-A KUBE-") && strings.Contains(line, "-j KUBE-") {
+				deleteRuleCmds = append(deleteRuleCmds, fmt.Sprintf("sudo iptables -t %s -D %s || true", table, line[3:]))
+			}
+			// Flush and delete all KUBE chains
+			if strings.HasPrefix(line, ":KUBE-") {
+				chain := strings.Split(line, " ")[0][1:]
+				deleteRuleCmds = append(deleteRuleCmds, fmt.Sprintf("sudo iptables -t %s -F %s || true", table, chain))
+				deleteChainCmds = append(deleteChainCmds, fmt.Sprintf("sudo iptables -t %s -X %s || true", table, chain))
+			}
+		}
+		cmd := strings.Join(append(deleteRuleCmds, deleteChainCmds...), "\n")
+
+		ginkgo.By("deleting all KUBE-* iptables chains")
+		result, err = e2essh.SSH(cmd, host, framework.TestContext.Provider)
+		if err != nil || result.Code != 0 {
+			e2essh.LogResult(result)
+			framework.Failf("couldn't delete iptable rules: %v", err)
+		}
+
+		ginkgo.By("verifying that kube-proxy rules are eventually recreated")
+		framework.ExpectNoError(e2eservice.VerifyServeHostnameServiceUp(f.ClientSet, ns, host, podNames, svcIP, servicePort))
+
+		ginkgo.By("verifying that kubelet rules are eventually recreated")
+		err = utilwait.PollImmediate(framework.Poll, framework.RestartNodeReadyAgainTimeout, func() (bool, error) {
+			result, err = e2essh.SSH("sudo iptables-save -t nat", host, framework.TestContext.Provider)
+			if err != nil || result.Code != 0 {
+				e2essh.LogResult(result)
+				return false, err
+			}
+
+			if strings.Contains(result.Stdout, "\n-A KUBE-MARK-DROP ") {
+				return true, nil
+			}
+			return false, nil
+		})
+		framework.ExpectNoError(err, "kubelet did not recreate its iptables rules")
+	})
 })

test/e2e/network/service.go

@@ -476,18 +476,6 @@ var _ = SIGDescribe("Services", func() {
 		}
 		framework.ExpectNoError(e2eservice.VerifyServeHostnameServiceUp(cs, ns, host, podNames1, svc1IP, servicePort))
 		framework.ExpectNoError(e2eservice.VerifyServeHostnameServiceUp(cs, ns, host, podNames2, svc2IP, servicePort))
-
-		ginkgo.By("Removing iptable rules")
-		result, err := e2essh.SSH(`
-			sudo iptables -t nat -F KUBE-SERVICES || true;
-			sudo iptables -t nat -F KUBE-PORTALS-HOST || true;
-			sudo iptables -t nat -F KUBE-PORTALS-CONTAINER || true`, host, framework.TestContext.Provider)
-		if err != nil || result.Code != 0 {
-			e2essh.LogResult(result)
-			framework.Failf("couldn't remove iptable rules: %v", err)
-		}
-		framework.ExpectNoError(e2eservice.VerifyServeHostnameServiceUp(cs, ns, host, podNames1, svc1IP, servicePort))
-		framework.ExpectNoError(e2eservice.VerifyServeHostnameServiceUp(cs, ns, host, podNames2, svc2IP, servicePort))
 	})
 
 	ginkgo.It("should work after restarting apiserver [Disruptive]", func() {