From df8f83fd519360f7a2e3e9abe3796d9247555d59 Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Thu, 29 Sep 2016 09:31:52 +0200 Subject: [PATCH] Use multiple clients in load test to avoid "no cached connections" errors. --- pkg/client/restclient/plugin_test.go | 2 +- pkg/client/restclient/transport.go | 10 ++-- test/e2e/load.go | 76 +++++++++++++++++++++++++--- 3 files changed, 75 insertions(+), 13 deletions(-) diff --git a/pkg/client/restclient/plugin_test.go b/pkg/client/restclient/plugin_test.go index 76b6dd7ea9e..42226fba5b6 100644 --- a/pkg/client/restclient/plugin_test.go +++ b/pkg/client/restclient/plugin_test.go @@ -61,7 +61,7 @@ func TestAuthPluginWrapTransport(t *testing.T) { if len(tc.plugin) != 0 { c.AuthProvider = &clientcmdapi.AuthProviderConfig{Name: tc.plugin} } - tConfig, err := c.transportConfig() + tConfig, err := c.TransportConfig() if err != nil { // Unknown/bad plugins are expected to fail here. if !tc.expectErr { diff --git a/pkg/client/restclient/transport.go b/pkg/client/restclient/transport.go index c385914e290..5f3d671d655 100644 --- a/pkg/client/restclient/transport.go +++ b/pkg/client/restclient/transport.go @@ -26,7 +26,7 @@ import ( // TLSConfigFor returns a tls.Config that will provide the transport level security defined // by the provided Config. Will return nil if no transport level security is requested. func TLSConfigFor(config *Config) (*tls.Config, error) { - cfg, err := config.transportConfig() + cfg, err := config.TransportConfig() if err != nil { return nil, err } @@ -37,7 +37,7 @@ func TLSConfigFor(config *Config) (*tls.Config, error) { // or transport level security defined by the provided Config. Will return the // default http.DefaultTransport if no special case behavior is needed. func TransportFor(config *Config) (http.RoundTripper, error) { - cfg, err := config.transportConfig() + cfg, err := config.TransportConfig() if err != nil { return nil, err } @@ -49,15 +49,15 @@ func TransportFor(config *Config) (http.RoundTripper, error) { // the underlying connection (like WebSocket or HTTP2 clients). Pure HTTP clients should use // the higher level TransportFor or RESTClientFor methods. func HTTPWrappersForConfig(config *Config, rt http.RoundTripper) (http.RoundTripper, error) { - cfg, err := config.transportConfig() + cfg, err := config.TransportConfig() if err != nil { return nil, err } return transport.HTTPWrappersForConfig(cfg, rt) } -// transportConfig converts a client config to an appropriate transport config. -func (c *Config) transportConfig() (*transport.Config, error) { +// TransportConfig converts a client config to an appropriate transport config. +func (c *Config) TransportConfig() (*transport.Config, error) { wt := c.WrapTransport if c.AuthProvider != nil { provider, err := GetAuthProvider(c.Host, c.AuthProvider, c.AuthConfigPersister) diff --git a/test/e2e/load.go b/test/e2e/load.go index 315d6bf82b4..cb588562f01 100644 --- a/test/e2e/load.go +++ b/test/e2e/load.go @@ -20,6 +20,8 @@ import ( "fmt" "math" "math/rand" + "net" + "net/http" "os" "strconv" "sync" @@ -28,9 +30,12 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" unversionedcore "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/unversioned" + "k8s.io/kubernetes/pkg/client/restclient" + "k8s.io/kubernetes/pkg/client/transport" client "k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/util/intstr" + utilnet "k8s.io/kubernetes/pkg/util/net" "k8s.io/kubernetes/test/e2e/framework" . "github.com/onsi/ginkgo" @@ -140,7 +145,7 @@ var _ = framework.KubeDescribe("Load capacity", func() { namespaces = createNamespaces(f, nodeCount, itArg.podsPerNode) totalPods := itArg.podsPerNode * nodeCount - configs = generateRCConfigs(totalPods, itArg.image, itArg.command, c, namespaces) + configs = generateRCConfigs(totalPods, itArg.image, itArg.command, namespaces) var services []*api.Service // Read the environment variable to see if we want to create services createServices := os.Getenv("CREATE_SERVICES") @@ -218,6 +223,52 @@ func createNamespaces(f *framework.Framework, nodeCount, podsPerNode int) []*api return namespaces } +func createClients(numberOfClients int) ([]*client.Client, error) { + clients := make([]*client.Client, numberOfClients) + for i := 0; i < numberOfClients; i++ { + config, err := framework.LoadConfig() + Expect(err).NotTo(HaveOccurred()) + config.QPS = 100 + config.Burst = 200 + if framework.TestContext.KubeAPIContentType != "" { + config.ContentType = framework.TestContext.KubeAPIContentType + } + + // For the purpose of this test, we want to force that clients + // do not share underlying transport (which is a default behavior + // in Kubernetes). Thus, we are explicitly creating transport for + // each client here. + transportConfig, err := config.TransportConfig() + if err != nil { + return nil, err + } + tlsConfig, err := transport.TLSConfigFor(transportConfig) + if err != nil { + return nil, err + } + config.Transport = utilnet.SetTransportDefaults(&http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: tlsConfig, + MaxIdleConnsPerHost: 100, + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial, + }) + // Overwrite TLS-related fields from config to avoid collision with + // Transport field. + config.TLSClientConfig = restclient.TLSClientConfig{} + + c, err := client.New(config) + if err != nil { + return nil, err + } + clients[i] = c + } + return clients, nil +} + func computeRCCounts(total int) (int, int, int) { // Small RCs owns ~0.5 of total number of pods, medium and big RCs ~0.25 each. // For example for 3000 pods (100 nodes, 30 pods per node) there are: @@ -232,22 +283,33 @@ func computeRCCounts(total int) (int, int, int) { return smallRCCount, mediumRCCount, bigRCCount } -func generateRCConfigs(totalPods int, image string, command []string, c *client.Client, nss []*api.Namespace) []*framework.RCConfig { +func generateRCConfigs(totalPods int, image string, command []string, nss []*api.Namespace) []*framework.RCConfig { configs := make([]*framework.RCConfig, 0) smallRCCount, mediumRCCount, bigRCCount := computeRCCounts(totalPods) - configs = append(configs, generateRCConfigsForGroup(c, nss, smallRCGroupName, smallRCSize, smallRCCount, image, command)...) - configs = append(configs, generateRCConfigsForGroup(c, nss, mediumRCGroupName, mediumRCSize, mediumRCCount, image, command)...) - configs = append(configs, generateRCConfigsForGroup(c, nss, bigRCGroupName, bigRCSize, bigRCCount, image, command)...) + configs = append(configs, generateRCConfigsForGroup(nss, smallRCGroupName, smallRCSize, smallRCCount, image, command)...) + configs = append(configs, generateRCConfigsForGroup(nss, mediumRCGroupName, mediumRCSize, mediumRCCount, image, command)...) + configs = append(configs, generateRCConfigsForGroup(nss, bigRCGroupName, bigRCSize, bigRCCount, image, command)...) + + // Create a number of clients to better simulate real usecase + // where not everyone is using exactly the same client. + rcsPerClient := 20 + clients, err := createClients((len(configs) + rcsPerClient - 1) / rcsPerClient) + framework.ExpectNoError(err) + + for i := 0; i < len(configs); i++ { + configs[i].Client = clients[i%len(clients)] + } return configs } -func generateRCConfigsForGroup(c *client.Client, nss []*api.Namespace, groupName string, size, count int, image string, command []string) []*framework.RCConfig { +func generateRCConfigsForGroup( + nss []*api.Namespace, groupName string, size, count int, image string, command []string) []*framework.RCConfig { configs := make([]*framework.RCConfig, 0, count) for i := 1; i <= count; i++ { config := &framework.RCConfig{ - Client: c, + Client: nil, // this will be overwritten later Name: groupName + "-" + strconv.Itoa(i), Namespace: nss[i%len(nss)].Name, Timeout: 10 * time.Minute,