diff --git a/build/root/WORKSPACE b/build/root/WORKSPACE index e411ef563fe..e7f23f182c5 100644 --- a/build/root/WORKSPACE +++ b/build/root/WORKSPACE @@ -12,6 +12,16 @@ http_archive( urls = ["https://github.com/kubernetes/repo-infra/archive/9dedd5f4093884c133ad5ea73695b28338b954ab.tar.gz"], ) +ETCD_VERSION = "3.0.17" + +new_http_archive( + name = "com_coreos_etcd", + build_file = "third_party/etcd.BUILD", + sha256 = "274c46a7f8d26f7ae99d6880610f54933cbcf7f3beafa19236c52eb5df8c7a0b", + strip_prefix = "etcd-v%s-linux-amd64" % ETCD_VERSION, + urls = ["https://github.com/coreos/etcd/releases/download/v%s/etcd-v%s-linux-amd64.tar.gz" % (ETCD_VERSION, ETCD_VERSION)], +) + # This contains a patch to not prepend ./ to tarfiles produced by pkg_tar. # When merged upstream, we'll no longer need to use ixdy's fork: # https://bazel-review.googlesource.com/#/c/10390/ diff --git a/test/integration/etcd/etcd_storage_path_test.go b/test/integration/etcd/etcd_storage_path_test.go index fd17823947d..fccf080609a 100644 --- a/test/integration/etcd/etcd_storage_path_test.go +++ b/test/integration/etcd/etcd_storage_path_test.go @@ -588,7 +588,7 @@ func startRealMasterOrDie(t *testing.T, certDir string) (*allClient, clientv3.KV kubeAPIServerOptions := options.NewServerRunOptions() kubeAPIServerOptions.SecureServing.BindAddress = net.ParseIP("127.0.0.1") kubeAPIServerOptions.SecureServing.ServerCert.CertDirectory = certDir - kubeAPIServerOptions.Etcd.StorageConfig.ServerList = []string{framework.GetEtcdURLFromEnv()} + kubeAPIServerOptions.Etcd.StorageConfig.ServerList = []string{framework.GetEtcdURL()} kubeAPIServerOptions.Etcd.DefaultStorageMediaType = runtime.ContentTypeJSON // TODO use protobuf? kubeAPIServerOptions.ServiceClusterIPRange = *defaultServiceClusterIPRange kubeAPIServerOptions.Authorization.Mode = "RBAC" diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index 2c34a30329b..671f4cc84a2 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -102,7 +102,7 @@ func TestAggregatedAPIServer(t *testing.T) { kubeAPIServerOptions.SecureServing.BindPort = kubePort kubeAPIServerOptions.SecureServing.ServerCert.CertDirectory = certDir kubeAPIServerOptions.InsecureServing.BindPort = 0 - kubeAPIServerOptions.Etcd.StorageConfig.ServerList = []string{framework.GetEtcdURLFromEnv()} + kubeAPIServerOptions.Etcd.StorageConfig.ServerList = []string{framework.GetEtcdURL()} kubeAPIServerOptions.ServiceClusterIPRange = *defaultServiceClusterIPRange kubeAPIServerOptions.Authentication.RequestHeader.UsernameHeaders = []string{"X-Remote-User"} kubeAPIServerOptions.Authentication.RequestHeader.GroupHeaders = []string{"X-Remote-Group"} @@ -190,7 +190,7 @@ func TestAggregatedAPIServer(t *testing.T) { "--requestheader-allowed-names=kube-aggregator", "--authentication-kubeconfig", kubeconfigFile.Name(), "--authorization-kubeconfig", kubeconfigFile.Name(), - "--etcd-servers", framework.GetEtcdURLFromEnv(), + "--etcd-servers", framework.GetEtcdURL(), "--cert-dir", wardleCertDir, }) if err := wardleCmd.Execute(); err != nil { @@ -266,7 +266,7 @@ func TestAggregatedAPIServer(t *testing.T) { "--core-kubeconfig", kubeconfigFile.Name(), "--authentication-kubeconfig", kubeconfigFile.Name(), "--authorization-kubeconfig", kubeconfigFile.Name(), - "--etcd-servers", framework.GetEtcdURLFromEnv(), + "--etcd-servers", framework.GetEtcdURL(), "--cert-dir", aggregatorCertDir, }) if err := aggregatorCmd.Execute(); err != nil { diff --git a/test/integration/federation/framework/api.go b/test/integration/federation/framework/api.go index 07df7f09a7e..f1d79cd56dd 100644 --- a/test/integration/federation/framework/api.go +++ b/test/integration/federation/framework/api.go @@ -36,7 +36,7 @@ const apiNoun = "federation apiserver" // GetRunOptions returns the default run options that can be used to run a test federation apiserver. func GetRunOptions() *options.ServerRunOptions { r := options.NewServerRunOptions() - r.Etcd.StorageConfig.ServerList = []string{framework.GetEtcdURLFromEnv()} + r.Etcd.StorageConfig.ServerList = []string{framework.GetEtcdURL()} // Use a unique prefix to ensure isolation from other tests using the same etcd instance r.Etcd.StorageConfig.Prefix = uuid.New() // Disable secure serving diff --git a/test/integration/framework/BUILD b/test/integration/framework/BUILD index 25b6e766239..ab99ef886d6 100644 --- a/test/integration/framework/BUILD +++ b/test/integration/framework/BUILD @@ -10,10 +10,14 @@ load( go_library( name = "go_default_library", srcs = [ + "etcd.go", "master_utils.go", "perf_utils.go", "serializer.go", ], + data = [ + "@com_coreos_etcd//:etcd", + ], tags = ["automanaged"], deps = [ "//pkg/api:go_default_library", diff --git a/test/integration/framework/etcd.go b/test/integration/framework/etcd.go new file mode 100644 index 00000000000..dcb4bbc4e23 --- /dev/null +++ b/test/integration/framework/etcd.go @@ -0,0 +1,109 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "fmt" + "hash/adler32" + "io" + "io/ioutil" + "math/rand" + "os" + "os/exec" + "path/filepath" + "sync" + + "k8s.io/kubernetes/pkg/util/env" + + "github.com/golang/glog" +) + +var ( + etcdSetup sync.Once + etcdURL = "" +) + +func setupETCD() { + etcdSetup.Do(func() { + if os.Getenv("RUNFILES_DIR") == "" { + etcdURL = env.GetEnvAsStringOrFallback("KUBE_INTEGRATION_ETCD_URL", "http://127.0.0.1:2379") + return + } + etcdPath := filepath.Join(os.Getenv("RUNFILES_DIR"), "com_coreos_etcd/etcd") + // give every test the same random port each run + etcdPort := 20000 + rand.New(rand.NewSource(int64(adler32.Checksum([]byte(os.Args[0]))))).Intn(5000) + etcdURL = fmt.Sprintf("http://127.0.0.1:%d", etcdPort) + + info, err := os.Stat(etcdPath) + if err != nil { + glog.Fatalf("Unable to stat etcd: %v", err) + } + if info.IsDir() { + glog.Fatalf("Did not expect %q to be a directory", etcdPath) + } + + etcdDataDir, err := ioutil.TempDir(os.TempDir(), "integration_test_etcd_data") + if err != nil { + glog.Fatalf("Unable to make temp etcd data dir: %v", err) + } + glog.Infof("storing etcd data in: %v", etcdDataDir) + + etcdCmd := exec.Command( + etcdPath, + "--data-dir", + etcdDataDir, + "--listen-client-urls", + GetEtcdURL(), + "--advertise-client-urls", + GetEtcdURL(), + "--listen-peer-urls", + "http://127.0.0.1:0", + ) + + stdout, err := etcdCmd.StdoutPipe() + if err != nil { + glog.Fatalf("Failed to run etcd: %v", err) + } + stderr, err := etcdCmd.StderrPipe() + if err != nil { + glog.Fatalf("Failed to run etcd: %v", err) + } + if err := etcdCmd.Start(); err != nil { + glog.Fatalf("Failed to run etcd: %v", err) + } + + go io.Copy(os.Stdout, stdout) + go io.Copy(os.Stderr, stderr) + + go func() { + if err := etcdCmd.Wait(); err != nil { + glog.Fatalf("Failed to run etcd: %v", err) + } + glog.Fatalf("etcd should not have succeeded") + }() + }) +} + +func EtcdMain(tests func() int) { + setupETCD() + os.Exit(tests()) +} + +// return the EtcdURL +func GetEtcdURL() string { + return etcdURL +} diff --git a/test/integration/framework/master_utils.go b/test/integration/framework/master_utils.go index 5de1d866ca7..83485d1d382 100644 --- a/test/integration/framework/master_utils.go +++ b/test/integration/framework/master_utils.go @@ -70,7 +70,6 @@ import ( "k8s.io/kubernetes/pkg/kubectl" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" "k8s.io/kubernetes/pkg/master" - "k8s.io/kubernetes/pkg/util/env" "k8s.io/kubernetes/pkg/version" "k8s.io/kubernetes/plugin/pkg/admission/admit" ) @@ -298,20 +297,13 @@ func parseCIDROrDie(cidr string) *net.IPNet { return parsed } -// return the EtcdURL -func GetEtcdURLFromEnv() string { - url := env.GetEnvAsStringOrFallback("KUBE_INTEGRATION_ETCD_URL", "http://127.0.0.1:2379") - glog.V(4).Infof("Using KUBE_INTEGRATION_ETCD_URL=%q", url) - return url -} - // Returns a basic master config. func NewMasterConfig() *master.Config { // This causes the integration tests to exercise the etcd // prefix code, so please don't change without ensuring // sufficient coverage in other ways. etcdOptions := options.NewEtcdOptions(storagebackend.NewDefaultConfig(uuid.New(), api.Scheme, nil)) - etcdOptions.StorageConfig.ServerList = []string{GetEtcdURLFromEnv()} + etcdOptions.StorageConfig.ServerList = []string{GetEtcdURL()} info, _ := runtime.SerializerInfoForMediaType(api.Codecs.SupportedMediaTypes(), runtime.ContentTypeJSON) ns := NewSingleContentTypeSerializer(api.Scheme, info) diff --git a/third_party/etcd.BUILD b/third_party/etcd.BUILD new file mode 100644 index 00000000000..816ffa75254 --- /dev/null +++ b/third_party/etcd.BUILD @@ -0,0 +1 @@ +exports_files(["etcd"])