Switch e2e_node to etcd3

staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json

@@ -38,10 +38,6 @@
 			"ImportPath": "github.com/coreos/etcd/auth/authpb",
 			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
 		},
-		{
-			"ImportPath": "github.com/coreos/etcd/client",
-			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
-		},
 		{
 			"ImportPath": "github.com/coreos/etcd/clientv3",
 			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
@@ -58,14 +54,6 @@
 			"ImportPath": "github.com/coreos/etcd/mvcc/mvccpb",
 			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
 		},
-		{
-			"ImportPath": "github.com/coreos/etcd/pkg/pathutil",
-			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
-		},
-		{
-			"ImportPath": "github.com/coreos/etcd/pkg/srv",
-			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
-		},
 		{
 			"ImportPath": "github.com/coreos/etcd/pkg/tlsutil",
 			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
@@ -78,14 +66,6 @@
 			"ImportPath": "github.com/coreos/etcd/pkg/types",
 			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
 		},
-		{
-			"ImportPath": "github.com/coreos/etcd/version",
-			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
-		},
-		{
-			"ImportPath": "github.com/coreos/go-semver/semver",
-			"Rev": "568e959cd89871e61434c1143528d9162da89ef2"
-		},
 		{
 			"ImportPath": "github.com/coreos/go-systemd/daemon",
 			"Rev": "48702e0da86bd25e76cfef347e2adeb434a0d0a6"
@@ -322,10 +302,6 @@
 			"ImportPath": "github.com/stretchr/testify/assert",
 			"Rev": "c679ae2cc0cb27ec3293fea7e254e47386f05d69"
 		},
-		{
-			"ImportPath": "github.com/ugorji/go/codec",
-			"Rev": "ded73eae5db7e7a0ef6f55aace87a2873c5d2b74"
-		},
 		{
 			"ImportPath": "golang.org/x/crypto/ssh/terminal",
 			"Rev": "de0752318171da717af4ce24d0a2e8626afaeb11"
@@ -1162,10 +1138,6 @@
 			"ImportPath": "k8s.io/apiserver/pkg/storage/etcd/metrics",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
-		{
-			"ImportPath": "k8s.io/apiserver/pkg/storage/etcd/util",
-			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-		},
 		{
 			"ImportPath": "k8s.io/apiserver/pkg/storage/etcd3",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json

@@ -38,10 +38,6 @@
 			"ImportPath": "github.com/coreos/etcd/auth/authpb",
 			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
 		},
-		{
-			"ImportPath": "github.com/coreos/etcd/client",
-			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
-		},
 		{
 			"ImportPath": "github.com/coreos/etcd/clientv3",
 			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
@@ -58,14 +54,6 @@
 			"ImportPath": "github.com/coreos/etcd/mvcc/mvccpb",
 			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
 		},
-		{
-			"ImportPath": "github.com/coreos/etcd/pkg/pathutil",
-			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
-		},
-		{
-			"ImportPath": "github.com/coreos/etcd/pkg/srv",
-			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
-		},
 		{
 			"ImportPath": "github.com/coreos/etcd/pkg/tlsutil",
 			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
@@ -78,14 +66,6 @@
 			"ImportPath": "github.com/coreos/etcd/pkg/types",
 			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
 		},
-		{
-			"ImportPath": "github.com/coreos/etcd/version",
-			"Rev": "420a452267a7ce45b3fcbed04d54030d69964fc1"
-		},
-		{
-			"ImportPath": "github.com/coreos/go-semver/semver",
-			"Rev": "568e959cd89871e61434c1143528d9162da89ef2"
-		},
 		{
 			"ImportPath": "github.com/coreos/go-systemd/daemon",
 			"Rev": "48702e0da86bd25e76cfef347e2adeb434a0d0a6"
@@ -302,10 +282,6 @@
 			"ImportPath": "github.com/spf13/pflag",
 			"Rev": "583c0c0531f06d5278b7d917446061adc344b5cd"
 		},
-		{
-			"ImportPath": "github.com/ugorji/go/codec",
-			"Rev": "ded73eae5db7e7a0ef6f55aace87a2873c5d2b74"
-		},
 		{
 			"ImportPath": "golang.org/x/crypto/ssh/terminal",
 			"Rev": "de0752318171da717af4ce24d0a2e8626afaeb11"
@@ -1126,10 +1102,6 @@
 			"ImportPath": "k8s.io/apiserver/pkg/storage/etcd/metrics",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
-		{
-			"ImportPath": "k8s.io/apiserver/pkg/storage/etcd/util",
-			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-		},
 		{
 			"ImportPath": "k8s.io/apiserver/pkg/storage/etcd3",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

test/e2e_node/e2e_node_suite_test.go

@@ -86,7 +86,7 @@ const rootfs = "/rootfs"
 func TestE2eNode(t *testing.T) {
 	if *runServicesMode {
 		// If run-services-mode is specified, only run services in current process.
-		services.RunE2EServices()
+		services.RunE2EServices(t)
 		return
 	}
 	if *runKubeletMode {

test/e2e_node/services/BUILD

@@ -9,7 +9,6 @@ go_library(
     name = "go_default_library",
     srcs = [
         "apiserver.go",
-        "etcd.go",
         "internal_services.go",
         "kubelet.go",
         "logs.go",
@@ -29,6 +28,8 @@ go_library(
         "//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//staging/src/k8s.io/client-go/dynamic:go_default_library",
@@ -39,10 +40,6 @@ go_library(
         "//test/e2e/framework:go_default_library",
         "//test/e2e_node/builder:go_default_library",
         "//test/e2e_node/remote:go_default_library",
-        "//vendor/github.com/coreos/etcd/etcdserver:go_default_library",
-        "//vendor/github.com/coreos/etcd/etcdserver/api/v2http:go_default_library",
-        "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library",
-        "//vendor/github.com/coreos/etcd/pkg/types:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/kardianos/osext:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",

test/e2e_node/services/apiserver.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net"
 
+	"k8s.io/apiserver/pkg/storage/storagebackend"
 	apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 )
@@ -31,21 +32,23 @@ const (
 )
 
 // APIServer is a server which manages apiserver.
-type APIServer struct{}
+type APIServer struct {
+	storageConfig storagebackend.Config
+	stopCh        chan struct{}
+}
 
 // NewAPIServer creates an apiserver.
-func NewAPIServer() *APIServer {
+func NewAPIServer(storageConfig storagebackend.Config) *APIServer {
-	return &APIServer{}
+	return &APIServer{
+		storageConfig: storageConfig,
+		stopCh:        make(chan struct{}),
+	}
 }
 
 // Start starts the apiserver, returns when apiserver is ready.
 func (a *APIServer) Start() error {
 	o := options.NewServerRunOptions()
-	o.Etcd.StorageConfig.ServerList = []string{getEtcdClientURL()}
+	o.Etcd.StorageConfig = a.storageConfig
-	// TODO: Current setup of etcd in e2e-node tests doesn't support etcd v3
-	// protocol. We should migrate it to use the same infrastructure as all
-	// other tests (pkg/storage/etcd/testing).
-	o.Etcd.StorageConfig.Type = "etcd2"
 	_, ipnet, err := net.ParseCIDR(clusterIPRange)
 	if err != nil {
 		return err
@@ -56,14 +59,12 @@ func (a *APIServer) Start() error {
 	errCh := make(chan error)
 	go func() {
 		defer close(errCh)
-		stopCh := make(chan struct{})
-		defer close(stopCh)
 		completedOptions, err := apiserver.Complete(o)
 		if err != nil {
 			errCh <- fmt.Errorf("set apiserver default options error: %v", err)
 			return
 		}
-		err = apiserver.Run(completedOptions, stopCh)
+		err = apiserver.Run(completedOptions, a.stopCh)
 		if err != nil {
 			errCh <- fmt.Errorf("run apiserver error: %v", err)
 			return
@@ -80,6 +81,10 @@ func (a *APIServer) Start() error {
 // Stop stops the apiserver. Currently, there is no way to stop the apiserver.
 // The function is here only for completion.
 func (a *APIServer) Stop() error {
+	if a.stopCh != nil {
+		close(a.stopCh)
+		a.stopCh = nil
+	}
 	return nil
 }
 

test/e2e_node/services/etcd.go

@@ -1,161 +0,0 @@
-/*
-Copyright 2016 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package services
-
-import (
-	"crypto/tls"
-	"net"
-	"net/http"
-	"net/url"
-	"time"
-
-	"github.com/coreos/etcd/etcdserver"
-	"github.com/coreos/etcd/etcdserver/api/v2http"
-	"github.com/coreos/etcd/pkg/transport"
-	"github.com/coreos/etcd/pkg/types"
-	"github.com/golang/glog"
-)
-
-// TODO: These tests should not be leveraging v2http
-// TODO(random-liu): Add service interface to manage services with the same behaviour.
-
-// All following configurations are got from etcd source code.
-// TODO(random-liu): Use embed.NewConfig after etcd3 is supported.
-const (
-	etcdName           = "etcd"
-	clientURLStr       = "http://localhost:4001" // clientURL has listener created and handles etcd API traffic
-	peerURLStr         = "http://localhost:7001" // peerURL does't have listener created, it is used to pass Etcd validation
-	snapCount          = etcdserver.DefaultSnapCount
-	maxSnapFiles       = 5
-	maxWALFiles        = 5
-	tickMs             = 100
-	electionTicks      = 10
-	etcdHealthCheckURL = clientURLStr + "/v2/keys/" // Trailing slash is required,
-)
-
-// EtcdServer is a server which manages etcd.
-type EtcdServer struct {
-	*etcdserver.EtcdServer
-	config       *etcdserver.ServerConfig
-	clientListen net.Listener
-}
-
-// NewEtcd creates a new default etcd server using 'dataDir' for persistence.
-func NewEtcd(dataDir string) *EtcdServer {
-	clientURLs, err := types.NewURLs([]string{clientURLStr})
-	if err != nil {
-		glog.Fatalf("Failed to parse client url %q: %v", clientURLStr, err)
-	}
-	peerURLs, err := types.NewURLs([]string{peerURLStr})
-	if err != nil {
-		glog.Fatalf("Failed to parse peer url %q: %v", peerURLStr, err)
-	}
-
-	config := &etcdserver.ServerConfig{
-		Name:               etcdName,
-		ClientURLs:         clientURLs,
-		PeerURLs:           peerURLs,
-		DataDir:            dataDir,
-		InitialPeerURLsMap: map[string]types.URLs{etcdName: peerURLs},
-		NewCluster:         true,
-		SnapCount:          snapCount,
-		MaxSnapFiles:       maxSnapFiles,
-		MaxWALFiles:        maxWALFiles,
-		TickMs:             tickMs,
-		ElectionTicks:      electionTicks,
-		AuthToken:          "simple",
-	}
-
-	return &EtcdServer{
-		config: config,
-	}
-}
-
-// Start starts the etcd server and listening for client connections
-func (e *EtcdServer) Start() error {
-	var err error
-	e.EtcdServer, err = etcdserver.NewServer(e.config)
-	if err != nil {
-		return err
-	}
-	// create client listener, there should be only one url
-	e.clientListen, err = createListener(e.config.ClientURLs[0])
-	if err != nil {
-		return err
-	}
-
-	// start etcd
-	e.EtcdServer.Start()
-
-	// setup client listener
-	ch := v2http.NewClientHandler(e.EtcdServer, e.config.ReqTimeout())
-	errCh := make(chan error)
-	go func(l net.Listener) {
-		defer close(errCh)
-		srv := &http.Server{
-			Handler:     ch,
-			ReadTimeout: 5 * time.Minute,
-		}
-		// Serve always returns a non-nil error.
-		errCh <- srv.Serve(l)
-	}(e.clientListen)
-
-	err = readinessCheck("etcd", []string{etcdHealthCheckURL}, errCh)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// Stop closes all connections and stops the Etcd server
-func (e *EtcdServer) Stop() error {
-	if e.EtcdServer != nil {
-		e.EtcdServer.Stop()
-	}
-	if e.clientListen != nil {
-		err := e.clientListen.Close()
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// Name returns the server's unique name
-func (e *EtcdServer) Name() string {
-	return etcdName
-}
-
-func createListener(url url.URL) (net.Listener, error) {
-	l, err := net.Listen("tcp", url.Host)
-	if err != nil {
-		return nil, err
-	}
-	l, err = transport.NewKeepAliveListener(l, url.Scheme, &tls.Config{})
-	if err != nil {
-		return nil, err
-	}
-	return l, nil
-}
-
-func getEtcdClientURL() string {
-	return clientURLStr
-}
-
-func getEtcdHealthCheckURL() string {
-	return etcdHealthCheckURL
-}

test/e2e_node/services/internal_services.go

@@ -17,9 +17,11 @@ limitations under the License.
 package services
 
 import (
-	"io/ioutil"
 	"os"
+	"testing"
 
+	etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	"github.com/golang/glog"
@@ -29,7 +31,8 @@ import (
 type e2eServices struct {
 	rmDirs []string
 	// statically linked e2e services
-	etcdServer   *EtcdServer
+	etcdServer   *etcdtesting.EtcdTestServer
+	etcdStorage  *storagebackend.Config
 	apiServer    *APIServer
 	nsController *NamespaceController
 }
@@ -40,9 +43,9 @@ func newE2EServices() *e2eServices {
 
 // run starts all e2e services and wait for the termination signal. Once receives the
 // termination signal, it will stop the e2e services gracefully.
-func (es *e2eServices) run() error {
+func (es *e2eServices) run(t *testing.T) error {
-	defer es.stop()
+	defer es.stop(t)
-	if err := es.start(); err != nil {
+	if err := es.start(t); err != nil {
 		return err
 	}
 	// Wait until receiving a termination signal.
@@ -51,13 +54,13 @@ func (es *e2eServices) run() error {
 }
 
 // start starts the tests embedded services or returns an error.
-func (es *e2eServices) start() error {
+func (es *e2eServices) start(t *testing.T) error {
 	glog.Info("Starting e2e services...")
-	err := es.startEtcd()
+	err := es.startEtcd(t)
 	if err != nil {
 		return err
 	}
-	err = es.startApiServer()
+	err = es.startApiServer(es.etcdStorage)
 	if err != nil {
 		return err
 	}
@@ -70,7 +73,7 @@ func (es *e2eServices) start() error {
 }
 
 // stop stops the embedded e2e services.
-func (es *e2eServices) stop() {
+func (es *e2eServices) stop(t *testing.T) {
 	glog.Info("Stopping e2e services...")
 	// TODO(random-liu): Use a loop to stop all services after introducing
 	// service interface.
@@ -90,9 +93,7 @@ func (es *e2eServices) stop() {
 
 	glog.Info("Stopping etcd")
 	if es.etcdServer != nil {
-		if err := es.etcdServer.Stop(); err != nil {
+		es.etcdServer.Terminate(t)
-			glog.Errorf("Failed to stop %q: %v", es.etcdServer.Name(), err)
-		}
 	}
 
 	for _, d := range es.rmDirs {
@@ -107,23 +108,18 @@ func (es *e2eServices) stop() {
 }
 
 // startEtcd starts the embedded etcd instance or returns an error.
-func (es *e2eServices) startEtcd() error {
+func (es *e2eServices) startEtcd(t *testing.T) error {
 	glog.Info("Starting etcd")
-	// Create data directory in current working space.
+	server, etcdStorage := etcdtesting.NewUnsecuredEtcd3TestClientServer(t)
-	dataDir, err := ioutil.TempDir(".", "etcd")
+	es.etcdServer = server
-	if err != nil {
+	es.etcdStorage = etcdStorage
-		return err
+	return nil
-	}
-	// Mark the dataDir as directories to remove.
-	es.rmDirs = append(es.rmDirs, dataDir)
-	es.etcdServer = NewEtcd(dataDir)
-	return es.etcdServer.Start()
 }
 
 // startApiServer starts the embedded API server or returns an error.
-func (es *e2eServices) startApiServer() error {
+func (es *e2eServices) startApiServer(etcdStorage *storagebackend.Config) error {
 	glog.Info("Starting API server")
-	es.apiServer = NewAPIServer()
+	es.apiServer = NewAPIServer(*etcdStorage)
 	return es.apiServer.Start()
 }
 
@@ -137,7 +133,6 @@ func (es *e2eServices) startNamespaceController() error {
 // getServicesHealthCheckURLs returns the health check urls for the internal services.
 func getServicesHealthCheckURLs() []string {
 	return []string{
-		getEtcdHealthCheckURL(),
 		getAPIServerHealthCheckURL(),
 	}
 }

test/e2e_node/services/services.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"testing"
 
 	"github.com/golang/glog"
 	"github.com/kardianos/osext"
@@ -105,12 +106,12 @@ func (e *E2EServices) Stop() {
 
 // RunE2EServices actually start the e2e services. This function is used to
 // start e2e services in current process. This is only used in run-services-mode.
-func RunE2EServices() {
+func RunE2EServices(t *testing.T) {
 	// Populate global DefaultFeatureGate with value from TestContext.FeatureGates.
 	// This way, statically-linked components see the same feature gate config as the test context.
 	utilfeature.DefaultFeatureGate.SetFromMap(framework.TestContext.FeatureGates)
 	e := newE2EServices()
-	if err := e.run(); err != nil {
+	if err := e.run(t); err != nil {
 		glog.Fatalf("Failed to run e2e services: %v", err)
 	}
 }