From 7398c83afdfbf0497b8b924b9af6af40ab6c6f2b Mon Sep 17 00:00:00 2001 From: Kenichi Omichi Date: Tue, 24 Mar 2020 20:59:08 +0000 Subject: [PATCH] Move RestartApiserver() into e2e/network The function is called from e2e/network test only, so this moves the function into the test for reducing e2e/framework/util.go code and removing invalid dependency on e2e test framework. --- test/e2e/framework/nodes_util.go | 5 +- test/e2e/framework/util.go | 80 ------------------------------ test/e2e/network/service.go | 85 +++++++++++++++++++++++++++++++- 3 files changed, 87 insertions(+), 83 deletions(-) diff --git a/test/e2e/framework/nodes_util.go b/test/e2e/framework/nodes_util.go index 161f234f891..34f7383e409 100644 --- a/test/e2e/framework/nodes_util.go +++ b/test/e2e/framework/nodes_util.go @@ -52,7 +52,7 @@ func MasterUpgrade(f *Framework, v string) error { case "gce": return masterUpgradeGCE(v, false) case "gke": - return masterUpgradeGKE(f.Namespace.Name, v) + return MasterUpgradeGKE(f.Namespace.Name, v) case "kubernetes-anywhere": return masterUpgradeKubernetesAnywhere(v) default: @@ -113,7 +113,8 @@ func appendContainerCommandGroupIfNeeded(args []string) []string { return args } -func masterUpgradeGKE(namespace string, v string) error { +// MasterUpgradeGKE upgrades master node to the specified version on GKE. +func MasterUpgradeGKE(namespace string, v string) error { Logf("Upgrading master to %q", v) args := []string{ "container", diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index d2fe977ef73..13e8cc80a1b 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -1115,86 +1115,6 @@ func AllNodesReady(c clientset.Interface, timeout time.Duration) error { return nil } -// RestartApiserver restarts the kube-apiserver. -func RestartApiserver(namespace string, cs clientset.Interface) error { - // TODO: Make it work for all providers. - if !ProviderIs("gce", "gke", "aws") { - return fmt.Errorf("unsupported provider for RestartApiserver: %s", TestContext.Provider) - } - if ProviderIs("gce", "aws") { - initialRestartCount, err := getApiserverRestartCount(cs) - if err != nil { - return fmt.Errorf("failed to get apiserver's restart count: %v", err) - } - if err := sshRestartMaster(); err != nil { - return fmt.Errorf("failed to restart apiserver: %v", err) - } - return waitForApiserverRestarted(cs, initialRestartCount) - } - // GKE doesn't allow ssh access, so use a same-version master - // upgrade to teardown/recreate master. - v, err := cs.Discovery().ServerVersion() - if err != nil { - return err - } - return masterUpgradeGKE(namespace, v.GitVersion[1:]) // strip leading 'v' -} - -func sshRestartMaster() error { - if !ProviderIs("gce", "aws") { - return fmt.Errorf("unsupported provider for sshRestartMaster: %s", TestContext.Provider) - } - var command string - if ProviderIs("gce") { - command = "pidof kube-apiserver | xargs sudo kill" - } else { - command = "sudo /etc/init.d/kube-apiserver restart" - } - Logf("Restarting master via ssh, running: %v", command) - result, err := e2essh.SSH(command, net.JoinHostPort(GetMasterHost(), sshPort), TestContext.Provider) - if err != nil || result.Code != 0 { - e2essh.LogResult(result) - return fmt.Errorf("couldn't restart apiserver: %v", err) - } - return nil -} - -// waitForApiserverRestarted waits until apiserver's restart count increased. -func waitForApiserverRestarted(c clientset.Interface, initialRestartCount int32) error { - for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) { - restartCount, err := getApiserverRestartCount(c) - if err != nil { - Logf("Failed to get apiserver's restart count: %v", err) - continue - } - if restartCount > initialRestartCount { - Logf("Apiserver has restarted.") - return nil - } - Logf("Waiting for apiserver restart count to increase") - } - return fmt.Errorf("timed out waiting for apiserver to be restarted") -} - -func getApiserverRestartCount(c clientset.Interface) (int32, error) { - label := labels.SelectorFromSet(labels.Set(map[string]string{"component": "kube-apiserver"})) - listOpts := metav1.ListOptions{LabelSelector: label.String()} - pods, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), listOpts) - if err != nil { - return -1, err - } - if len(pods.Items) != 1 { - return -1, fmt.Errorf("unexpected number of apiserver pod: %d", len(pods.Items)) - } - for _, s := range pods.Items[0].Status.ContainerStatuses { - if s.Name != "kube-apiserver" { - continue - } - return s.RestartCount, nil - } - return -1, fmt.Errorf("Failed to find kube-apiserver container in pod") -} - // RestartControllerManager restarts the kube-controller-manager. func RestartControllerManager() error { // TODO: Make it work for all providers and distros. diff --git a/test/e2e/network/service.go b/test/e2e/network/service.go index 13239e99c97..781995db99c 100644 --- a/test/e2e/network/service.go +++ b/test/e2e/network/service.go @@ -84,6 +84,9 @@ const ( // AffinityConfirmCount is the number of needed continuous requests to confirm that // affinity is enabled. AffinityConfirmCount = 15 + + // ssh port + sshPort = "22" ) var ( @@ -1107,7 +1110,7 @@ var _ = SIGDescribe("Services", func() { // Restart apiserver ginkgo.By("Restarting apiserver") - if err := framework.RestartApiserver(ns, cs); err != nil { + if err := restartApiserver(ns, cs); err != nil { framework.Failf("error restarting apiserver: %v", err) } ginkgo.By("Waiting for apiserver to come up by polling /healthz") @@ -3431,3 +3434,83 @@ func validateEndpointsPorts(c clientset.Interface, namespace, serviceName string } return fmt.Errorf("Timed out waiting for service %s in namespace %s to expose endpoints %v (%v elapsed)", serviceName, namespace, expectedEndpoints, framework.ServiceStartTimeout) } + +// restartApiserver restarts the kube-apiserver. +func restartApiserver(namespace string, cs clientset.Interface) error { + // TODO: Make it work for all providers. + if !framework.ProviderIs("gce", "gke", "aws") { + return fmt.Errorf("unsupported provider for RestartApiserver: %s", framework.TestContext.Provider) + } + if framework.ProviderIs("gce", "aws") { + initialRestartCount, err := getApiserverRestartCount(cs) + if err != nil { + return fmt.Errorf("failed to get apiserver's restart count: %v", err) + } + if err := sshRestartMaster(); err != nil { + return fmt.Errorf("failed to restart apiserver: %v", err) + } + return waitForApiserverRestarted(cs, initialRestartCount) + } + // GKE doesn't allow ssh access, so use a same-version master + // upgrade to teardown/recreate master. + v, err := cs.Discovery().ServerVersion() + if err != nil { + return err + } + return framework.MasterUpgradeGKE(namespace, v.GitVersion[1:]) // strip leading 'v' +} + +func sshRestartMaster() error { + if !framework.ProviderIs("gce", "aws") { + return fmt.Errorf("unsupported provider for sshRestartMaster: %s", framework.TestContext.Provider) + } + var command string + if framework.ProviderIs("gce") { + command = "pidof kube-apiserver | xargs sudo kill" + } else { + command = "sudo /etc/init.d/kube-apiserver restart" + } + framework.Logf("Restarting master via ssh, running: %v", command) + result, err := e2essh.SSH(command, net.JoinHostPort(framework.GetMasterHost(), sshPort), framework.TestContext.Provider) + if err != nil || result.Code != 0 { + e2essh.LogResult(result) + return fmt.Errorf("couldn't restart apiserver: %v", err) + } + return nil +} + +// waitForApiserverRestarted waits until apiserver's restart count increased. +func waitForApiserverRestarted(c clientset.Interface, initialRestartCount int32) error { + for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) { + restartCount, err := getApiserverRestartCount(c) + if err != nil { + framework.Logf("Failed to get apiserver's restart count: %v", err) + continue + } + if restartCount > initialRestartCount { + framework.Logf("Apiserver has restarted.") + return nil + } + framework.Logf("Waiting for apiserver restart count to increase") + } + return fmt.Errorf("timed out waiting for apiserver to be restarted") +} + +func getApiserverRestartCount(c clientset.Interface) (int32, error) { + label := labels.SelectorFromSet(labels.Set(map[string]string{"component": "kube-apiserver"})) + listOpts := metav1.ListOptions{LabelSelector: label.String()} + pods, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), listOpts) + if err != nil { + return -1, err + } + if len(pods.Items) != 1 { + return -1, fmt.Errorf("unexpected number of apiserver pod: %d", len(pods.Items)) + } + for _, s := range pods.Items[0].Status.ContainerStatuses { + if s.Name != "kube-apiserver" { + continue + } + return s.RestartCount, nil + } + return -1, fmt.Errorf("Failed to find kube-apiserver container in pod") +}