From 1d83b28dfe24e38f894be4c76d1318338149b971 Mon Sep 17 00:00:00 2001 From: jornshen Date: Tue, 15 Sep 2020 21:05:30 +0800 Subject: [PATCH] refector service some e2e cases to make it runing in multi providers the cases are as fellow: - "should work after restarting kube-proxy" - "should work after restarting apiserver" --- test/e2e/framework/pod/resource.go | 11 ++ test/e2e/framework/skipper/BUILD | 1 + test/e2e/framework/skipper/skipper.go | 23 ++++ test/e2e/network/service.go | 165 ++++++-------------------- 4 files changed, 74 insertions(+), 126 deletions(-) diff --git a/test/e2e/framework/pod/resource.go b/test/e2e/framework/pod/resource.go index e092910cb3b..cde443eb81e 100644 --- a/test/e2e/framework/pod/resource.go +++ b/test/e2e/framework/pod/resource.go @@ -587,6 +587,17 @@ func GetPodsInNamespace(c clientset.Interface, ns string, ignoreLabels map[strin return filtered, nil } +// GetPods return the label matched pods in the given ns +func GetPods(c clientset.Interface, ns string, matchLabels map[string]string) ([]v1.Pod, error) { + label := labels.SelectorFromSet(matchLabels) + listOpts := metav1.ListOptions{LabelSelector: label.String()} + pods, err := c.CoreV1().Pods(ns).List(context.TODO(), listOpts) + if err != nil { + return []v1.Pod{}, err + } + return pods.Items, nil +} + // GetPodSecretUpdateTimeout returns the timeout duration for updating pod secret. func GetPodSecretUpdateTimeout(c clientset.Interface) time.Duration { // With SecretManager(ConfigMapManager), we may have to wait up to full sync period + diff --git a/test/e2e/framework/skipper/BUILD b/test/e2e/framework/skipper/BUILD index ee4f535f426..f6899207c0b 100644 --- a/test/e2e/framework/skipper/BUILD +++ b/test/e2e/framework/skipper/BUILD @@ -8,6 +8,7 @@ go_library( deps = [ "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", diff --git a/test/e2e/framework/skipper/skipper.go b/test/e2e/framework/skipper/skipper.go index ff0dec92f77..76d679649f7 100644 --- a/test/e2e/framework/skipper/skipper.go +++ b/test/e2e/framework/skipper/skipper.go @@ -30,6 +30,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" utilversion "k8s.io/apimachinery/pkg/util/version" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -285,3 +286,25 @@ func RunIfSystemSpecNameIs(names ...string) { } skipInternalf(1, "Skipped because system spec name %q is not in %v", framework.TestContext.SystemSpecName, names) } + +// SkipUnlessComponentRunsAsPodsAndClientCanDeleteThem run if the component run as pods and client can delete them +func SkipUnlessComponentRunsAsPodsAndClientCanDeleteThem(componentName string, c clientset.Interface, ns string, labelSet labels.Set) { + // verify if component run as pod + label := labels.SelectorFromSet(labelSet) + listOpts := metav1.ListOptions{LabelSelector: label.String()} + pods, err := c.CoreV1().Pods(ns).List(context.TODO(), listOpts) + framework.Logf("SkipUnlessComponentRunsAsPodsAndClientCanDeleteThem: %v, %v", pods, err) + if err != nil { + skipInternalf(1, "Skipped because client failed to get component:%s pod err:%v", componentName, err) + } + + if len(pods.Items) == 0 { + skipInternalf(1, "Skipped because component:%s is not running as pod.", componentName) + } + + // verify if client can delete pod + pod := pods.Items[0] + if err := c.CoreV1().Pods(ns).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}}); err != nil { + skipInternalf(1, "Skipped because client failed to delete component:%s pod, err:%v", componentName, err) + } +} diff --git a/test/e2e/network/service.go b/test/e2e/network/service.go index 8be4656be47..b135c22e1bc 100644 --- a/test/e2e/network/service.go +++ b/test/e2e/network/service.go @@ -60,7 +60,6 @@ import ( e2erc "k8s.io/kubernetes/test/e2e/framework/rc" e2eservice "k8s.io/kubernetes/test/e2e/framework/service" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" - e2essh "k8s.io/kubernetes/test/e2e/framework/ssh" "k8s.io/kubernetes/test/e2e/storage/utils" testutils "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" @@ -83,6 +82,12 @@ const ( // AffinityConfirmCount is the number of needed continuous requests to confirm that // affinity is enabled. AffinityConfirmCount = 15 + + // label define which is used to find kube-proxy and kube-apiserver pod + kubeProxyLabelName = "kube-proxy" + clusterAddonLabelKey = "k8s-app" + kubeAPIServerLabelName = "kube-apiserver" + clusterComponentKey = "component" ) var ( @@ -703,45 +708,8 @@ func getServeHostnameService(name string) *v1.Service { return svc } -// restartKubeProxy restarts kube-proxy on the given host. -func restartKubeProxy(host string) error { - // TODO: Make it work for all providers. - if !framework.ProviderIs("gce", "gke", "aws") { - return fmt.Errorf("unsupported provider for restartKubeProxy: %s", framework.TestContext.Provider) - } - // kubelet will restart the kube-proxy since it's running in a static pod - framework.Logf("Killing kube-proxy on node %v", host) - result, err := e2essh.SSH("sudo pkill kube-proxy", host, framework.TestContext.Provider) - if err != nil || result.Code != 0 { - e2essh.LogResult(result) - return fmt.Errorf("couldn't restart kube-proxy: %v", err) - } - // wait for kube-proxy to come back up - sshCmd := "sudo /bin/sh -c 'pgrep kube-proxy | wc -l'" - err = wait.Poll(5*time.Second, 60*time.Second, func() (bool, error) { - framework.Logf("Waiting for kubeproxy to come back up with %v on %v", sshCmd, host) - result, err := e2essh.SSH(sshCmd, host, framework.TestContext.Provider) - if err != nil { - return false, err - } - if result.Code != 0 { - e2essh.LogResult(result) - return false, fmt.Errorf("failed to run command, exited %d", result.Code) - } - if result.Stdout == "0\n" { - return false, nil - } - framework.Logf("kube-proxy is back up.") - return true, nil - }) - if err != nil { - return fmt.Errorf("kube-proxy didn't recover: %v", err) - } - return nil -} - -// waitForApiserverUp waits for the kube-apiserver to be up. -func waitForApiserverUp(c clientset.Interface) error { +// waitForAPIServerUp waits for the kube-apiserver to be up. +func waitForAPIServerUp(c clientset.Interface) error { for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) { body, err := c.CoreV1().RESTClient().Get().AbsPath("/healthz").Do(context.TODO()).Raw() if err == nil && string(body) == "ok" { @@ -1101,10 +1069,10 @@ var _ = SIGDescribe("Services", func() { }) ginkgo.It("should work after restarting kube-proxy [Disruptive]", func() { - // TODO: use the ServiceTestJig here - e2eskipper.SkipUnlessProviderIs("gce", "gke") - e2eskipper.SkipUnlessSSHKeyPresent() + kubeProxyLabelSet := map[string]string{clusterAddonLabelKey: kubeProxyLabelName} + e2eskipper.SkipUnlessComponentRunsAsPodsAndClientCanDeleteThem(kubeProxyLabelName, cs, metav1.NamespaceSystem, kubeProxyLabelSet) + // TODO: use the ServiceTestJig here ns := f.Namespace.Name numPods, servicePort := 3, defaultServeHostnameServicePort @@ -1127,18 +1095,10 @@ var _ = SIGDescribe("Services", func() { framework.Failf("VIPs conflict: %v", svc1IP) } - hosts, err := e2essh.NodeSSHHosts(cs) - framework.ExpectNoError(err, "failed to find external/internal IPs for every node") - if len(hosts) == 0 { - framework.Failf("No ssh-able nodes") - } - host := hosts[0] - framework.ExpectNoError(verifyServeHostnameServiceUp(cs, ns, podNames1, svc1IP, servicePort)) framework.ExpectNoError(verifyServeHostnameServiceUp(cs, ns, podNames2, svc2IP, servicePort)) - ginkgo.By(fmt.Sprintf("Restarting kube-proxy on %v", host)) - if err := restartKubeProxy(host); err != nil { + if err := restartComponent(cs, kubeProxyLabelName, metav1.NamespaceSystem, kubeProxyLabelSet); err != nil { framework.Failf("error restarting kube-proxy: %v", err) } framework.ExpectNoError(verifyServeHostnameServiceUp(cs, ns, podNames1, svc1IP, servicePort)) @@ -1146,12 +1106,14 @@ var _ = SIGDescribe("Services", func() { }) ginkgo.It("should work after restarting apiserver [Disruptive]", func() { - // TODO: use the ServiceTestJig here - e2eskipper.SkipUnlessProviderIs("gce", "gke") - e2eskipper.SkipUnlessSSHKeyPresent() + if !framework.ProviderIs("gke") { + e2eskipper.SkipUnlessComponentRunsAsPodsAndClientCanDeleteThem(kubeAPIServerLabelName, cs, metav1.NamespaceSystem, map[string]string{clusterComponentKey: kubeAPIServerLabelName}) + } + + // TODO: use the ServiceTestJig here ns := f.Namespace.Name - numPods, servicePort := 3, 80 + numPods, servicePort := 3, defaultServeHostnameServicePort svc1 := "restart-apiserver-1" svc2 := "restart-apiserver-2" @@ -1170,7 +1132,7 @@ var _ = SIGDescribe("Services", func() { framework.Failf("error restarting apiserver: %v", err) } ginkgo.By("Waiting for apiserver to come up by polling /healthz") - if err := waitForApiserverUp(cs); err != nil { + if err := waitForAPIServerUp(cs); err != nil { framework.Failf("error while waiting for apiserver up: %v", err) } framework.ExpectNoError(verifyServeHostnameServiceUp(cs, ns, podNames1, svc1IP, servicePort)) @@ -3733,84 +3695,35 @@ func validateEndpointsPorts(c clientset.Interface, namespace, serviceName string return nil } -// restartApiserver restarts the kube-apiserver. func restartApiserver(namespace string, cs clientset.Interface) error { - // TODO: Make it work for all providers. - if !framework.ProviderIs("gce", "gke", "aws") { - return fmt.Errorf("unsupported provider for RestartApiserver: %s", framework.TestContext.Provider) - } - if framework.ProviderIs("gce", "aws") { - initialRestartCount, err := getApiserverRestartCount(cs) + if framework.ProviderIs("gke") { + // GKE use a same-version master upgrade to teardown/recreate master. + v, err := cs.Discovery().ServerVersion() if err != nil { - return fmt.Errorf("failed to get apiserver's restart count: %v", err) + return err } - if err := sshRestartMaster(); err != nil { - return fmt.Errorf("failed to restart apiserver: %v", err) - } - return waitForApiserverRestarted(cs, initialRestartCount) + return framework.MasterUpgradeGKE(namespace, v.GitVersion[1:]) // strip leading 'v' } - // GKE doesn't allow ssh access, so use a same-version master - // upgrade to teardown/recreate master. - v, err := cs.Discovery().ServerVersion() + + return restartComponent(cs, kubeAPIServerLabelName, metav1.NamespaceSystem, map[string]string{clusterComponentKey: kubeAPIServerLabelName}) +} + +// restartComponent restarts component static pod +func restartComponent(cs clientset.Interface, cName, ns string, matchLabels map[string]string) error { + pods, err := e2epod.GetPods(cs, ns, matchLabels) if err != nil { - return err + return fmt.Errorf("failed to get %s's pods, err: %v", cName, err) + } + if len(pods) == 0 { + return fmt.Errorf("%s pod count is 0", cName) } - return framework.MasterUpgradeGKE(namespace, v.GitVersion[1:]) // strip leading 'v' -} -func sshRestartMaster() error { - if !framework.ProviderIs("gce", "aws") { - return fmt.Errorf("unsupported provider for sshRestartMaster: %s", framework.TestContext.Provider) + if err := e2epod.DeletePodsWithGracePeriod(cs, pods, 0); err != nil { + return fmt.Errorf("failed to restart component: %s, err: %v", cName, err) } - var command string - if framework.ProviderIs("gce") { - command = "pidof kube-apiserver | xargs sudo kill" - } else { - command = "sudo /etc/init.d/kube-apiserver restart" - } - framework.Logf("Restarting master via ssh, running: %v", command) - result, err := e2essh.SSH(command, net.JoinHostPort(framework.APIAddress(), e2essh.SSHPort), framework.TestContext.Provider) - if err != nil || result.Code != 0 { - e2essh.LogResult(result) - return fmt.Errorf("couldn't restart apiserver: %v", err) - } - return nil -} -// waitForApiserverRestarted waits until apiserver's restart count increased. -func waitForApiserverRestarted(c clientset.Interface, initialRestartCount int32) error { - for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) { - restartCount, err := getApiserverRestartCount(c) - if err != nil { - framework.Logf("Failed to get apiserver's restart count: %v", err) - continue - } - if restartCount > initialRestartCount { - framework.Logf("Apiserver has restarted.") - return nil - } - framework.Logf("Waiting for apiserver restart count to increase") - } - return fmt.Errorf("timed out waiting for apiserver to be restarted") -} - -func getApiserverRestartCount(c clientset.Interface) (int32, error) { - label := labels.SelectorFromSet(labels.Set(map[string]string{"component": "kube-apiserver"})) - listOpts := metav1.ListOptions{LabelSelector: label.String()} - pods, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), listOpts) - if err != nil { - return -1, err - } - if len(pods.Items) != 1 { - return -1, fmt.Errorf("unexpected number of apiserver pod: %d", len(pods.Items)) - } - for _, s := range pods.Items[0].Status.ContainerStatuses { - if s.Name != "kube-apiserver" { - continue - } - return s.RestartCount, nil - } - return -1, fmt.Errorf("Failed to find kube-apiserver container in pod") + _, err = e2epod.PodsCreatedByLabel(cs, ns, cName, int32(len(pods)), labels.SelectorFromSet(matchLabels)) + return err } var _ = SIGDescribe("SCTP [Feature:SCTP] [LinuxOnly]", func() {