diff --git a/transport/cache.go b/transport/cache.go
index da22cdee5..3cc0ef011 100644
--- a/transport/cache.go
+++ b/transport/cache.go
@@ -59,7 +59,7 @@ func (c *tlsTransportCache) get(config *Config) (http.RoundTripper, error) {
 		return nil, err
 	}
 	// The options didn't require a custom TLS config
-	if tlsConfig == nil {
+	if tlsConfig == nil && config.Dial == nil {
 		return http.DefaultTransport, nil
 	}
 
@@ -88,5 +88,5 @@ func tlsConfigKey(c *Config) (string, error) {
 		return "", err
 	}
 	// Only include the things that actually affect the tls.Config
-	return fmt.Sprintf("%v/%x/%x/%x/%v", c.TLS.Insecure, c.TLS.CAData, c.TLS.CertData, c.TLS.KeyData, c.TLS.ServerName), nil
+	return fmt.Sprintf("%v/%x/%x/%x/%v/%v", c.TLS.Insecure, c.TLS.CAData, c.TLS.CertData, c.TLS.KeyData, c.TLS.ServerName, fmt.Sprintf("%p", c.Dial)), nil
 }
diff --git a/transport/cache_test.go b/transport/cache_test.go
index 81f428de0..d3d14099d 100644
--- a/transport/cache_test.go
+++ b/transport/cache_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package transport
 
 import (
+	"net"
 	"net/http"
 	"testing"
 )
@@ -53,6 +54,8 @@ func TestTLSConfigKey(t *testing.T) {
 	// Make sure config fields that affect the tls config affect the cache key
 	uniqueConfigurations := map[string]*Config{
 		"no tls":   {},
+		"dialer":   {Dial: net.Dial},
+		"dialer2":  {Dial: func(network, address string) (net.Conn, error) { return nil, nil }},
 		"insecure": {TLS: TLSConfig{Insecure: true}},
 		"cadata 1": {TLS: TLSConfig{CAData: []byte{1}}},
 		"cadata 2": {TLS: TLSConfig{CAData: []byte{2}}},
@@ -104,11 +107,6 @@ func TestTLSConfigKey(t *testing.T) {
 	}
 	for nameA, valueA := range uniqueConfigurations {
 		for nameB, valueB := range uniqueConfigurations {
-			// Don't compare to ourselves
-			if nameA == nameB {
-				continue
-			}
-
 			keyA, err := tlsConfigKey(valueA)
 			if err != nil {
 				t.Errorf("Unexpected error for %q: %v", nameA, err)
@@ -119,6 +117,15 @@ func TestTLSConfigKey(t *testing.T) {
 				t.Errorf("Unexpected error for %q: %v", nameB, err)
 				continue
 			}
+
+			// Make sure we get the same key on the same config
+			if nameA == nameB {
+				if keyA != keyB {
+					t.Errorf("Expected identical cache keys for %q and %q, got:\n\t%s\n\t%s", nameA, nameB, keyA, keyB)
+				}
+				continue
+			}
+
 			if keyA == keyB {
 				t.Errorf("Expected unique cache keys for %q and %q, got:\n\t%s\n\t%s", nameA, nameB, keyA, keyB)
 				continue
diff --git a/transport/transport.go b/transport/transport.go
index 15be0a3e6..c2bb7ae5e 100644
--- a/transport/transport.go
+++ b/transport/transport.go
@@ -52,7 +52,7 @@ func New(config *Config) (http.RoundTripper, error) {
 // TLSConfigFor returns a tls.Config that will provide the transport level security defined
 // by the provided Config. Will return nil if no transport level security is requested.
 func TLSConfigFor(c *Config) (*tls.Config, error) {
-	if !(c.HasCA() || c.HasCertAuth() || c.TLS.Insecure) {
+	if !(c.HasCA() || c.HasCertAuth() || c.TLS.Insecure || len(c.TLS.ServerName) > 0) {
 		return nil, nil
 	}
 	if c.HasCA() && c.TLS.Insecure {
diff --git a/transport/transport_test.go b/transport/transport_test.go
index 4d2d78f86..8de751562 100644
--- a/transport/transport_test.go
+++ b/transport/transport_test.go
@@ -101,6 +101,13 @@ func TestNew(t *testing.T) {
 			Config:  &Config{},
 		},
 
+		"server name": {
+			TLS: true,
+			Config: &Config{TLS: TLSConfig{
+				ServerName: "foo",
+			}},
+		},
+
 		"ca transport": {
 			TLS: true,
 			Config: &Config{