From ad46728158ff2dfc982e7c0cb16ddc0748b513b5 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Wed, 3 Oct 2018 01:39:57 -0400 Subject: [PATCH] Switch e2e_node to etcd3 --- .../k8s.io/kube-aggregator/Godeps/Godeps.json | 28 --- .../sample-apiserver/Godeps/Godeps.json | 28 --- test/e2e_node/e2e_node_suite_test.go | 2 +- test/e2e_node/services/BUILD | 7 +- test/e2e_node/services/apiserver.go | 27 +-- test/e2e_node/services/etcd.go | 161 ------------------ test/e2e_node/services/internal_services.go | 45 +++-- test/e2e_node/services/services.go | 5 +- 8 files changed, 42 insertions(+), 261 deletions(-) delete mode 100644 test/e2e_node/services/etcd.go diff --git a/staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json b/staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json index 4c5901100b6..0ea2b148616 100644 --- a/staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json +++ b/staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json @@ -38,10 +38,6 @@ "ImportPath": "github.com/coreos/etcd/auth/authpb", "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" }, - { - "ImportPath": "github.com/coreos/etcd/client", - "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" - }, { "ImportPath": "github.com/coreos/etcd/clientv3", "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" @@ -58,14 +54,6 @@ "ImportPath": "github.com/coreos/etcd/mvcc/mvccpb", "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" }, - { - "ImportPath": "github.com/coreos/etcd/pkg/pathutil", - "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" - }, - { - "ImportPath": "github.com/coreos/etcd/pkg/srv", - "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" - }, { "ImportPath": "github.com/coreos/etcd/pkg/tlsutil", "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" @@ -78,14 +66,6 @@ "ImportPath": "github.com/coreos/etcd/pkg/types", "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" }, - { - "ImportPath": "github.com/coreos/etcd/version", - "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" - }, - { - "ImportPath": "github.com/coreos/go-semver/semver", - "Rev": "568e959cd89871e61434c1143528d9162da89ef2" - }, { "ImportPath": "github.com/coreos/go-systemd/daemon", "Rev": "48702e0da86bd25e76cfef347e2adeb434a0d0a6" @@ -322,10 +302,6 @@ "ImportPath": "github.com/stretchr/testify/assert", "Rev": "c679ae2cc0cb27ec3293fea7e254e47386f05d69" }, - { - "ImportPath": "github.com/ugorji/go/codec", - "Rev": "ded73eae5db7e7a0ef6f55aace87a2873c5d2b74" - }, { "ImportPath": "golang.org/x/crypto/ssh/terminal", "Rev": "de0752318171da717af4ce24d0a2e8626afaeb11" @@ -1162,10 +1138,6 @@ "ImportPath": "k8s.io/apiserver/pkg/storage/etcd/metrics", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, - { - "ImportPath": "k8s.io/apiserver/pkg/storage/etcd/util", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, { "ImportPath": "k8s.io/apiserver/pkg/storage/etcd3", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" diff --git a/staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json b/staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json index 7003a50e3c3..4daa77c9b6d 100644 --- a/staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json +++ b/staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json @@ -38,10 +38,6 @@ "ImportPath": "github.com/coreos/etcd/auth/authpb", "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" }, - { - "ImportPath": "github.com/coreos/etcd/client", - "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" - }, { "ImportPath": "github.com/coreos/etcd/clientv3", "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" @@ -58,14 +54,6 @@ "ImportPath": "github.com/coreos/etcd/mvcc/mvccpb", "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" }, - { - "ImportPath": "github.com/coreos/etcd/pkg/pathutil", - "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" - }, - { - "ImportPath": "github.com/coreos/etcd/pkg/srv", - "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" - }, { "ImportPath": "github.com/coreos/etcd/pkg/tlsutil", "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" @@ -78,14 +66,6 @@ "ImportPath": "github.com/coreos/etcd/pkg/types", "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" }, - { - "ImportPath": "github.com/coreos/etcd/version", - "Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1" - }, - { - "ImportPath": "github.com/coreos/go-semver/semver", - "Rev": "568e959cd89871e61434c1143528d9162da89ef2" - }, { "ImportPath": "github.com/coreos/go-systemd/daemon", "Rev": "48702e0da86bd25e76cfef347e2adeb434a0d0a6" @@ -302,10 +282,6 @@ "ImportPath": "github.com/spf13/pflag", "Rev": "583c0c0531f06d5278b7d917446061adc344b5cd" }, - { - "ImportPath": "github.com/ugorji/go/codec", - "Rev": "ded73eae5db7e7a0ef6f55aace87a2873c5d2b74" - }, { "ImportPath": "golang.org/x/crypto/ssh/terminal", "Rev": "de0752318171da717af4ce24d0a2e8626afaeb11" @@ -1126,10 +1102,6 @@ "ImportPath": "k8s.io/apiserver/pkg/storage/etcd/metrics", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, - { - "ImportPath": "k8s.io/apiserver/pkg/storage/etcd/util", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, { "ImportPath": "k8s.io/apiserver/pkg/storage/etcd3", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index 91f3586ad4c..3cd0c01a7c1 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -86,7 +86,7 @@ const rootfs = "/rootfs" func TestE2eNode(t *testing.T) { if *runServicesMode { // If run-services-mode is specified, only run services in current process. - services.RunE2EServices() + services.RunE2EServices(t) return } if *runKubeletMode { diff --git a/test/e2e_node/services/BUILD b/test/e2e_node/services/BUILD index 56ca17f5fd6..60d41f37d6b 100644 --- a/test/e2e_node/services/BUILD +++ b/test/e2e_node/services/BUILD @@ -9,7 +9,6 @@ go_library( name = "go_default_library", srcs = [ "apiserver.go", - "etcd.go", "internal_services.go", "kubelet.go", "logs.go", @@ -29,6 +28,8 @@ go_library( "//pkg/kubelet/kubeletconfig/util/codec:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", "//staging/src/k8s.io/client-go/dynamic:go_default_library", @@ -39,10 +40,6 @@ go_library( "//test/e2e/framework:go_default_library", "//test/e2e_node/builder:go_default_library", "//test/e2e_node/remote:go_default_library", - "//vendor/github.com/coreos/etcd/etcdserver:go_default_library", - "//vendor/github.com/coreos/etcd/etcdserver/api/v2http:go_default_library", - "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library", - "//vendor/github.com/coreos/etcd/pkg/types:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/kardianos/osext:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", diff --git a/test/e2e_node/services/apiserver.go b/test/e2e_node/services/apiserver.go index 1d956fbecda..d66ff6e98c1 100644 --- a/test/e2e_node/services/apiserver.go +++ b/test/e2e_node/services/apiserver.go @@ -20,6 +20,7 @@ import ( "fmt" "net" + "k8s.io/apiserver/pkg/storage/storagebackend" apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" ) @@ -31,21 +32,23 @@ const ( ) // APIServer is a server which manages apiserver. -type APIServer struct{} +type APIServer struct { + storageConfig storagebackend.Config + stopCh chan struct{} +} // NewAPIServer creates an apiserver. -func NewAPIServer() *APIServer { - return &APIServer{} +func NewAPIServer(storageConfig storagebackend.Config) *APIServer { + return &APIServer{ + storageConfig: storageConfig, + stopCh: make(chan struct{}), + } } // Start starts the apiserver, returns when apiserver is ready. func (a *APIServer) Start() error { o := options.NewServerRunOptions() - o.Etcd.StorageConfig.ServerList = []string{getEtcdClientURL()} - // TODO: Current setup of etcd in e2e-node tests doesn't support etcd v3 - // protocol. We should migrate it to use the same infrastructure as all - // other tests (pkg/storage/etcd/testing). - o.Etcd.StorageConfig.Type = "etcd2" + o.Etcd.StorageConfig = a.storageConfig _, ipnet, err := net.ParseCIDR(clusterIPRange) if err != nil { return err @@ -56,14 +59,12 @@ func (a *APIServer) Start() error { errCh := make(chan error) go func() { defer close(errCh) - stopCh := make(chan struct{}) - defer close(stopCh) completedOptions, err := apiserver.Complete(o) if err != nil { errCh <- fmt.Errorf("set apiserver default options error: %v", err) return } - err = apiserver.Run(completedOptions, stopCh) + err = apiserver.Run(completedOptions, a.stopCh) if err != nil { errCh <- fmt.Errorf("run apiserver error: %v", err) return @@ -80,6 +81,10 @@ func (a *APIServer) Start() error { // Stop stops the apiserver. Currently, there is no way to stop the apiserver. // The function is here only for completion. func (a *APIServer) Stop() error { + if a.stopCh != nil { + close(a.stopCh) + a.stopCh = nil + } return nil } diff --git a/test/e2e_node/services/etcd.go b/test/e2e_node/services/etcd.go deleted file mode 100644 index 8c40dc7797e..00000000000 --- a/test/e2e_node/services/etcd.go +++ /dev/null @@ -1,161 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package services - -import ( - "crypto/tls" - "net" - "net/http" - "net/url" - "time" - - "github.com/coreos/etcd/etcdserver" - "github.com/coreos/etcd/etcdserver/api/v2http" - "github.com/coreos/etcd/pkg/transport" - "github.com/coreos/etcd/pkg/types" - "github.com/golang/glog" -) - -// TODO: These tests should not be leveraging v2http -// TODO(random-liu): Add service interface to manage services with the same behaviour. - -// All following configurations are got from etcd source code. -// TODO(random-liu): Use embed.NewConfig after etcd3 is supported. -const ( - etcdName = "etcd" - clientURLStr = "http://localhost:4001" // clientURL has listener created and handles etcd API traffic - peerURLStr = "http://localhost:7001" // peerURL does't have listener created, it is used to pass Etcd validation - snapCount = etcdserver.DefaultSnapCount - maxSnapFiles = 5 - maxWALFiles = 5 - tickMs = 100 - electionTicks = 10 - etcdHealthCheckURL = clientURLStr + "/v2/keys/" // Trailing slash is required, -) - -// EtcdServer is a server which manages etcd. -type EtcdServer struct { - *etcdserver.EtcdServer - config *etcdserver.ServerConfig - clientListen net.Listener -} - -// NewEtcd creates a new default etcd server using 'dataDir' for persistence. -func NewEtcd(dataDir string) *EtcdServer { - clientURLs, err := types.NewURLs([]string{clientURLStr}) - if err != nil { - glog.Fatalf("Failed to parse client url %q: %v", clientURLStr, err) - } - peerURLs, err := types.NewURLs([]string{peerURLStr}) - if err != nil { - glog.Fatalf("Failed to parse peer url %q: %v", peerURLStr, err) - } - - config := &etcdserver.ServerConfig{ - Name: etcdName, - ClientURLs: clientURLs, - PeerURLs: peerURLs, - DataDir: dataDir, - InitialPeerURLsMap: map[string]types.URLs{etcdName: peerURLs}, - NewCluster: true, - SnapCount: snapCount, - MaxSnapFiles: maxSnapFiles, - MaxWALFiles: maxWALFiles, - TickMs: tickMs, - ElectionTicks: electionTicks, - AuthToken: "simple", - } - - return &EtcdServer{ - config: config, - } -} - -// Start starts the etcd server and listening for client connections -func (e *EtcdServer) Start() error { - var err error - e.EtcdServer, err = etcdserver.NewServer(e.config) - if err != nil { - return err - } - // create client listener, there should be only one url - e.clientListen, err = createListener(e.config.ClientURLs[0]) - if err != nil { - return err - } - - // start etcd - e.EtcdServer.Start() - - // setup client listener - ch := v2http.NewClientHandler(e.EtcdServer, e.config.ReqTimeout()) - errCh := make(chan error) - go func(l net.Listener) { - defer close(errCh) - srv := &http.Server{ - Handler: ch, - ReadTimeout: 5 * time.Minute, - } - // Serve always returns a non-nil error. - errCh <- srv.Serve(l) - }(e.clientListen) - - err = readinessCheck("etcd", []string{etcdHealthCheckURL}, errCh) - if err != nil { - return err - } - return nil -} - -// Stop closes all connections and stops the Etcd server -func (e *EtcdServer) Stop() error { - if e.EtcdServer != nil { - e.EtcdServer.Stop() - } - if e.clientListen != nil { - err := e.clientListen.Close() - if err != nil { - return err - } - } - return nil -} - -// Name returns the server's unique name -func (e *EtcdServer) Name() string { - return etcdName -} - -func createListener(url url.URL) (net.Listener, error) { - l, err := net.Listen("tcp", url.Host) - if err != nil { - return nil, err - } - l, err = transport.NewKeepAliveListener(l, url.Scheme, &tls.Config{}) - if err != nil { - return nil, err - } - return l, nil -} - -func getEtcdClientURL() string { - return clientURLStr -} - -func getEtcdHealthCheckURL() string { - return etcdHealthCheckURL -} diff --git a/test/e2e_node/services/internal_services.go b/test/e2e_node/services/internal_services.go index 499b624ae34..356c00ef146 100644 --- a/test/e2e_node/services/internal_services.go +++ b/test/e2e_node/services/internal_services.go @@ -17,9 +17,11 @@ limitations under the License. package services import ( - "io/ioutil" "os" + "testing" + etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing" + "k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/kubernetes/test/e2e/framework" "github.com/golang/glog" @@ -29,7 +31,8 @@ import ( type e2eServices struct { rmDirs []string // statically linked e2e services - etcdServer *EtcdServer + etcdServer *etcdtesting.EtcdTestServer + etcdStorage *storagebackend.Config apiServer *APIServer nsController *NamespaceController } @@ -40,9 +43,9 @@ func newE2EServices() *e2eServices { // run starts all e2e services and wait for the termination signal. Once receives the // termination signal, it will stop the e2e services gracefully. -func (es *e2eServices) run() error { - defer es.stop() - if err := es.start(); err != nil { +func (es *e2eServices) run(t *testing.T) error { + defer es.stop(t) + if err := es.start(t); err != nil { return err } // Wait until receiving a termination signal. @@ -51,13 +54,13 @@ func (es *e2eServices) run() error { } // start starts the tests embedded services or returns an error. -func (es *e2eServices) start() error { +func (es *e2eServices) start(t *testing.T) error { glog.Info("Starting e2e services...") - err := es.startEtcd() + err := es.startEtcd(t) if err != nil { return err } - err = es.startApiServer() + err = es.startApiServer(es.etcdStorage) if err != nil { return err } @@ -70,7 +73,7 @@ func (es *e2eServices) start() error { } // stop stops the embedded e2e services. -func (es *e2eServices) stop() { +func (es *e2eServices) stop(t *testing.T) { glog.Info("Stopping e2e services...") // TODO(random-liu): Use a loop to stop all services after introducing // service interface. @@ -90,9 +93,7 @@ func (es *e2eServices) stop() { glog.Info("Stopping etcd") if es.etcdServer != nil { - if err := es.etcdServer.Stop(); err != nil { - glog.Errorf("Failed to stop %q: %v", es.etcdServer.Name(), err) - } + es.etcdServer.Terminate(t) } for _, d := range es.rmDirs { @@ -107,23 +108,18 @@ func (es *e2eServices) stop() { } // startEtcd starts the embedded etcd instance or returns an error. -func (es *e2eServices) startEtcd() error { +func (es *e2eServices) startEtcd(t *testing.T) error { glog.Info("Starting etcd") - // Create data directory in current working space. - dataDir, err := ioutil.TempDir(".", "etcd") - if err != nil { - return err - } - // Mark the dataDir as directories to remove. - es.rmDirs = append(es.rmDirs, dataDir) - es.etcdServer = NewEtcd(dataDir) - return es.etcdServer.Start() + server, etcdStorage := etcdtesting.NewUnsecuredEtcd3TestClientServer(t) + es.etcdServer = server + es.etcdStorage = etcdStorage + return nil } // startApiServer starts the embedded API server or returns an error. -func (es *e2eServices) startApiServer() error { +func (es *e2eServices) startApiServer(etcdStorage *storagebackend.Config) error { glog.Info("Starting API server") - es.apiServer = NewAPIServer() + es.apiServer = NewAPIServer(*etcdStorage) return es.apiServer.Start() } @@ -137,7 +133,6 @@ func (es *e2eServices) startNamespaceController() error { // getServicesHealthCheckURLs returns the health check urls for the internal services. func getServicesHealthCheckURLs() []string { return []string{ - getEtcdHealthCheckURL(), getAPIServerHealthCheckURL(), } } diff --git a/test/e2e_node/services/services.go b/test/e2e_node/services/services.go index 7a442d38e6a..b73658ab306 100644 --- a/test/e2e_node/services/services.go +++ b/test/e2e_node/services/services.go @@ -22,6 +22,7 @@ import ( "os" "os/exec" "path" + "testing" "github.com/golang/glog" "github.com/kardianos/osext" @@ -105,12 +106,12 @@ func (e *E2EServices) Stop() { // RunE2EServices actually start the e2e services. This function is used to // start e2e services in current process. This is only used in run-services-mode. -func RunE2EServices() { +func RunE2EServices(t *testing.T) { // Populate global DefaultFeatureGate with value from TestContext.FeatureGates. // This way, statically-linked components see the same feature gate config as the test context. utilfeature.DefaultFeatureGate.SetFromMap(framework.TestContext.FeatureGates) e := newE2EServices() - if err := e.run(); err != nil { + if err := e.run(t); err != nil { glog.Fatalf("Failed to run e2e services: %v", err) } }