From 42bb845f40229ef00db008e5adeef34a96841c25 Mon Sep 17 00:00:00 2001 From: Kenichi Omichi Date: Fri, 27 Mar 2020 03:43:33 +0000 Subject: [PATCH] Separate RestartControllerManager() as e2ekubesystem RestartControllerManager() is kube-controller specific function and it is better to separate the function as subpackage of e2e test framework. In addition, the function made invalid dependency into e2essh. So this separates the function into e2ekubesystem subpackage. --- test/e2e/framework/BUILD | 1 + test/e2e/framework/kubesystem/BUILD | 26 ++++++++ test/e2e/framework/kubesystem/kubesystem.go | 66 +++++++++++++++++++ test/e2e/framework/util.go | 38 ----------- test/e2e/network/BUILD | 1 + test/e2e/network/service.go | 9 +-- test/e2e/storage/BUILD | 1 + .../nfs_persistent_volume-disruptive.go | 5 +- 8 files changed, 103 insertions(+), 44 deletions(-) create mode 100644 test/e2e/framework/kubesystem/BUILD create mode 100644 test/e2e/framework/kubesystem/kubesystem.go diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index d46e7c3b962..7769ecde570 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -117,6 +117,7 @@ filegroup( "//test/e2e/framework/job:all-srcs", "//test/e2e/framework/kubectl:all-srcs", "//test/e2e/framework/kubelet:all-srcs", + "//test/e2e/framework/kubesystem:all-srcs", "//test/e2e/framework/log:all-srcs", "//test/e2e/framework/metrics:all-srcs", "//test/e2e/framework/network:all-srcs", diff --git a/test/e2e/framework/kubesystem/BUILD b/test/e2e/framework/kubesystem/BUILD new file mode 100644 index 00000000000..2e7c1659b9f --- /dev/null +++ b/test/e2e/framework/kubesystem/BUILD @@ -0,0 +1,26 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["kubesystem.go"], + importpath = "k8s.io/kubernetes/test/e2e/framework/kubesystem", + visibility = ["//visibility:public"], + deps = [ + "//test/e2e/framework:go_default_library", + "//test/e2e/framework/ssh:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/e2e/framework/kubesystem/kubesystem.go b/test/e2e/framework/kubesystem/kubesystem.go new file mode 100644 index 00000000000..bb7891184d3 --- /dev/null +++ b/test/e2e/framework/kubesystem/kubesystem.go @@ -0,0 +1,66 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubesystem + +import ( + "fmt" + "net" + "strconv" + "time" + + "k8s.io/kubernetes/test/e2e/framework" + e2essh "k8s.io/kubernetes/test/e2e/framework/ssh" +) + +const ( + // ssh port + sshPort = "22" +) + +// RestartControllerManager restarts the kube-controller-manager. +func RestartControllerManager() error { + // TODO: Make it work for all providers and distros. + if !framework.ProviderIs("gce", "aws") { + return fmt.Errorf("unsupported provider for RestartControllerManager: %s", framework.TestContext.Provider) + } + if framework.ProviderIs("gce") && !framework.MasterOSDistroIs("gci") { + return fmt.Errorf("unsupported master OS distro: %s", framework.TestContext.MasterOSDistro) + } + cmd := "pidof kube-controller-manager | xargs sudo kill" + framework.Logf("Restarting controller-manager via ssh, running: %v", cmd) + result, err := e2essh.SSH(cmd, net.JoinHostPort(framework.GetMasterHost(), sshPort), framework.TestContext.Provider) + if err != nil || result.Code != 0 { + e2essh.LogResult(result) + return fmt.Errorf("couldn't restart controller-manager: %v", err) + } + return nil +} + +// WaitForControllerManagerUp waits for the kube-controller-manager to be up. +func WaitForControllerManagerUp() error { + cmd := "curl http://localhost:" + strconv.Itoa(framework.InsecureKubeControllerManagerPort) + "/healthz" + for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) { + result, err := e2essh.SSH(cmd, net.JoinHostPort(framework.GetMasterHost(), sshPort), framework.TestContext.Provider) + if err != nil || result.Code != 0 { + e2essh.LogResult(result) + } + if result.Stdout == "ok" { + return nil + } + } + return fmt.Errorf("waiting for controller-manager timed out") +} diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index aff45e0b959..e8370f44be0 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -72,7 +72,6 @@ import ( e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - e2essh "k8s.io/kubernetes/test/e2e/framework/ssh" ) const ( @@ -143,9 +142,6 @@ const ( // TODO(justinsb): Avoid hardcoding this. awsMasterIP = "172.20.0.9" - - // ssh port - sshPort = "22" ) var ( @@ -1114,40 +1110,6 @@ func AllNodesReady(c clientset.Interface, timeout time.Duration) error { return nil } -// RestartControllerManager restarts the kube-controller-manager. -func RestartControllerManager() error { - // TODO: Make it work for all providers and distros. - if !ProviderIs("gce", "aws") { - return fmt.Errorf("unsupported provider for RestartControllerManager: %s", TestContext.Provider) - } - if ProviderIs("gce") && !MasterOSDistroIs("gci") { - return fmt.Errorf("unsupported master OS distro: %s", TestContext.MasterOSDistro) - } - cmd := "pidof kube-controller-manager | xargs sudo kill" - Logf("Restarting controller-manager via ssh, running: %v", cmd) - result, err := e2essh.SSH(cmd, net.JoinHostPort(GetMasterHost(), sshPort), TestContext.Provider) - if err != nil || result.Code != 0 { - e2essh.LogResult(result) - return fmt.Errorf("couldn't restart controller-manager: %v", err) - } - return nil -} - -// WaitForControllerManagerUp waits for the kube-controller-manager to be up. -func WaitForControllerManagerUp() error { - cmd := "curl http://localhost:" + strconv.Itoa(InsecureKubeControllerManagerPort) + "/healthz" - for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) { - result, err := e2essh.SSH(cmd, net.JoinHostPort(GetMasterHost(), sshPort), TestContext.Provider) - if err != nil || result.Code != 0 { - e2essh.LogResult(result) - } - if result.Stdout == "ok" { - return nil - } - } - return fmt.Errorf("waiting for controller-manager timed out") -} - // LookForStringInLog looks for the given string in the log of a specific pod container func LookForStringInLog(ns, podName, container, expectedString string, timeout time.Duration) (result string, err error) { return lookForString(expectedString, timeout, func() string { diff --git a/test/e2e/network/BUILD b/test/e2e/network/BUILD index 10d9af48453..f781bad8b7c 100644 --- a/test/e2e/network/BUILD +++ b/test/e2e/network/BUILD @@ -70,6 +70,7 @@ go_library( "//test/e2e/framework/deployment:go_default_library", "//test/e2e/framework/endpoints:go_default_library", "//test/e2e/framework/ingress:go_default_library", + "//test/e2e/framework/kubesystem:go_default_library", "//test/e2e/framework/network:go_default_library", "//test/e2e/framework/node:go_default_library", "//test/e2e/framework/pod:go_default_library", diff --git a/test/e2e/network/service.go b/test/e2e/network/service.go index aa3a0f9a7b3..3d7df617f46 100644 --- a/test/e2e/network/service.go +++ b/test/e2e/network/service.go @@ -47,6 +47,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2edeploy "k8s.io/kubernetes/test/e2e/framework/deployment" e2eendpoints "k8s.io/kubernetes/test/e2e/framework/endpoints" + e2ekubesystem "k8s.io/kubernetes/test/e2e/framework/kubesystem" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -2345,11 +2346,11 @@ var _ = SIGDescribe("Services", func() { } ginkgo.By("restart kube-controller-manager") - if err := framework.RestartControllerManager(); err != nil { - framework.Failf("framework.RestartControllerManager() = %v; want nil", err) + if err := e2ekubesystem.RestartControllerManager(); err != nil { + framework.Failf("e2ekubesystem.RestartControllerManager() = %v; want nil", err) } - if err := framework.WaitForControllerManagerUp(); err != nil { - framework.Failf("framework.WaitForControllerManagerUp() = %v; want nil", err) + if err := e2ekubesystem.WaitForControllerManagerUp(); err != nil { + framework.Failf("e2ekubesystem.WaitForControllerManagerUp() = %v; want nil", err) } ginkgo.By("health check should be reconciled") diff --git a/test/e2e/storage/BUILD b/test/e2e/storage/BUILD index f95cbed0341..ba7121fdccb 100644 --- a/test/e2e/storage/BUILD +++ b/test/e2e/storage/BUILD @@ -67,6 +67,7 @@ go_library( "//test/e2e/framework/auth:go_default_library", "//test/e2e/framework/deployment:go_default_library", "//test/e2e/framework/kubectl:go_default_library", + "//test/e2e/framework/kubesystem:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/e2e/framework/node:go_default_library", "//test/e2e/framework/pod:go_default_library", diff --git a/test/e2e/storage/nfs_persistent_volume-disruptive.go b/test/e2e/storage/nfs_persistent_volume-disruptive.go index 7083f2f3ccc..e2d3d522fb0 100644 --- a/test/e2e/storage/nfs_persistent_volume-disruptive.go +++ b/test/e2e/storage/nfs_persistent_volume-disruptive.go @@ -31,6 +31,7 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + e2ekubesystem "k8s.io/kubernetes/test/e2e/framework/kubesystem" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2epv "k8s.io/kubernetes/test/e2e/framework/pv" @@ -214,9 +215,9 @@ var _ = utils.SIGDescribe("NFSPersistentVolumes[Disruptive][Flaky]", func() { pvc2 = nil ginkgo.By("Restarting the kube-controller-manager") - err = framework.RestartControllerManager() + err = e2ekubesystem.RestartControllerManager() framework.ExpectNoError(err) - err = framework.WaitForControllerManagerUp() + err = e2ekubesystem.WaitForControllerManagerUp() framework.ExpectNoError(err) framework.Logf("kube-controller-manager restarted")