Merge pull request #16344 from thockin/enable-iptables-proxy

Auto commit by PR queue bot

cmd/kube-proxy/app/server.go

@@ -91,7 +91,7 @@ func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")
 	fs.Var(&s.PortRange, "proxy-port-range", "Range of host ports (beginPort-endPort, inclusive) that may be consumed in order to proxy service traffic. If unspecified (0-0) then ports will be randomly chosen.")
 	fs.StringVar(&s.HostnameOverride, "hostname-override", s.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.")
-	fs.StringVar(&s.ProxyMode, "proxy-mode", "", "Which proxy mode to use: 'userspace' (older, stable) or 'iptables' (experimental). If blank, look at the Node object on the Kubernetes API and respect the '"+experimentalProxyModeAnnotation+"' annotation if provided.  Otherwise use the best-available proxy (currently userspace, but may change in future versions).  If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.")
+	fs.StringVar(&s.ProxyMode, "proxy-mode", "", "Which proxy mode to use: 'userspace' (older) or 'iptables' (faster). If blank, look at the Node object on the Kubernetes API and respect the '"+experimentalProxyModeAnnotation+"' annotation if provided.  Otherwise use the best-available proxy (currently iptables).  If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.")
 	fs.DurationVar(&s.IptablesSyncPeriod, "iptables-sync-period", s.IptablesSyncPeriod, "How often iptables rules are refreshed (e.g. '5s', '1m', '2h22m').  Must be greater than 0.")
 	fs.DurationVar(&s.ConfigSyncPeriod, "config-sync-period", s.ConfigSyncPeriod, "How often configuration from the apiserver is refreshed.  Must be greater than 0.")
 	fs.BoolVar(&s.MasqueradeAll, "masquerade-all", false, "If using the pure iptables proxy, SNAT everything")
@@ -238,17 +238,8 @@ func NewProxyServerDefault(config *ProxyServerConfig) (*ProxyServer, error) {
 	var proxier proxy.ProxyProvider
 	var endpointsHandler proxyconfig.EndpointsConfigHandler
 
-	useIptablesProxy := false
+	proxyMode := getProxyMode(config.ProxyMode, client.Nodes(), hostname, iptInterface)
-	if mayTryIptablesProxy(config.ProxyMode, client.Nodes(), hostname) {
+	if proxyMode == proxyModeIptables {
-		var err error
-		// guaranteed false on error, error only necessary for debugging
-		useIptablesProxy, err = iptables.ShouldUseIptablesProxier()
-		if err != nil {
-			glog.Errorf("Can't determine whether to use iptables proxy, using userspace proxier: %v", err)
-		}
-	}
-
-	if useIptablesProxy {
 		glog.V(2).Info("Using iptables Proxier.")
 		proxierIptables, err := iptables.NewProxier(iptInterface, execer, config.IptablesSyncPeriod, config.MasqueradeAll)
 		if err != nil {
@@ -340,27 +331,28 @@ type nodeGetter interface {
 	Get(hostname string) (*api.Node, error)
 }
 
-func mayTryIptablesProxy(proxyMode string, client nodeGetter, hostname string) bool {
+func getProxyMode(proxyMode string, client nodeGetter, hostname string, iptver iptables.IptablesVersioner) string {
-	if proxyMode == proxyModeIptables {
+	if proxyMode == proxyModeUserspace {
-		glog.V(1).Infof("Flag proxy-mode allows iptables proxy")
+		return proxyModeUserspace
-		return true
+	} else if proxyMode == proxyModeIptables {
+		return tryIptablesProxy(iptver)
 	} else if proxyMode != "" {
-		glog.V(1).Infof("Flag proxy-mode=%q forbids iptables proxy", proxyMode)
+		glog.V(1).Infof("Flag proxy-mode=%q unknown, assuming iptables proxy", proxyMode)
-		return false
+		return tryIptablesProxy(iptver)
 	}
 	// proxyMode == "" - choose the best option.
 	if client == nil {
-		glog.Errorf("Not trying iptables proxy: nodeGetter is nil")
+		glog.Errorf("nodeGetter is nil: assuming iptables proxy")
-		return false
+		return tryIptablesProxy(iptver)
 	}
 	node, err := client.Get(hostname)
 	if err != nil {
-		glog.Errorf("Not trying iptables proxy: can't get Node %q: %v", hostname, err)
+		glog.Errorf("Can't get Node %q, assuming iptables proxy: %v", hostname, err)
-		return false
+		return tryIptablesProxy(iptver)
 	}
 	if node == nil {
-		glog.Errorf("Not trying iptables proxy: got nil Node %q", hostname)
+		glog.Errorf("Got nil Node %q, assuming iptables proxy: %v", hostname)
-		return false
+		return tryIptablesProxy(iptver)
 	}
 	proxyMode, found := node.Annotations[betaProxyModeAnnotation]
 	if found {
@@ -372,12 +364,27 @@ func mayTryIptablesProxy(proxyMode string, client nodeGetter, hostname string) b
 			glog.V(1).Infof("Found experimental annotation %q = %q", experimentalProxyModeAnnotation, proxyMode)
 		}
 	}
-	if proxyMode == proxyModeIptables {
+	if proxyMode == proxyModeUserspace {
-		glog.V(1).Infof("Annotation allows iptables proxy")
+		glog.V(1).Infof("Annotation demands userspace proxy")
-		return true
+		return proxyModeUserspace
 	}
-	glog.V(1).Infof("Not trying iptables proxy: %+v", node)
+	return tryIptablesProxy(iptver)
-	return false
+}
+
+func tryIptablesProxy(iptver iptables.IptablesVersioner) string {
+	var err error
+	// guaranteed false on error, error only necessary for debugging
+	useIptablesProxy, err := iptables.CanUseIptablesProxier(iptver)
+	if err != nil {
+		glog.Errorf("Can't determine whether to use iptables proxy, using userspace proxier: %v", err)
+		return proxyModeUserspace
+	}
+	if useIptablesProxy {
+		return proxyModeIptables
+	}
+	// Fallback.
+	glog.V(1).Infof("Can't use iptables proxy, using userspace proxier: %v", err)
+	return proxyModeUserspace
 }
 
 func (s *ProxyServer) birthCry() {

cmd/kube-proxy/app/server_test.go

@@ -17,10 +17,12 @@ limitations under the License.
 package app
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/util/iptables"
 )
 
 type fakeNodeInterface struct {
@@ -31,40 +33,176 @@ func (fake *fakeNodeInterface) Get(hostname string) (*api.Node, error) {
 	return &fake.node, nil
 }
 
-func Test_mayTryIptablesProxy(t *testing.T) {
+type fakeIptablesVersioner struct {
+	version string // what to return
+	err     error  // what to return
+}
+
+func (fake *fakeIptablesVersioner) GetVersion() (string, error) {
+	return fake.version, fake.err
+}
+
+func Test_getProxyMode(t *testing.T) {
 	var cases = []struct {
-		flag     string
+		flag            string
-		annKey   string
+		annotationKey   string
-		annVal   string
+		annotationVal   string
-		expected bool
+		iptablesVersion string
+		iptablesError   error
+		expected        string
 	}{
-		{"userspace", "", "", false},
+		{ // flag says userspace
-		{"iptables", "", "", true},
+			flag:     "userspace",
-		{"", "", "", false},
+			expected: proxyModeUserspace,
-		{"", "net.experimental.kubernetes.io/proxy-mode", "userspace", false},
+		},
-		{"", "net.experimental.kubernetes.io/proxy-mode", "iptables", true},
+		{ // flag says iptables, error detecting version
-		{"", "net.experimental.kubernetes.io/proxy-mode", "other", false},
+			flag:          "iptables",
-		{"", "net.experimental.kubernetes.io/proxy-mode", "", false},
+			iptablesError: fmt.Errorf("oops!"),
-		{"", "net.beta.kubernetes.io/proxy-mode", "userspace", false},
+			expected:      proxyModeUserspace,
-		{"", "net.beta.kubernetes.io/proxy-mode", "iptables", true},
+		},
-		{"", "net.beta.kubernetes.io/proxy-mode", "other", false},
+		{ // flag says iptables, version too low
-		{"", "net.beta.kubernetes.io/proxy-mode", "", false},
+			flag:            "iptables",
-		{"", "proxy-mode", "iptables", false},
+			iptablesVersion: "0.0.0",
-		{"userspace", "net.experimental.kubernetes.io/proxy-mode", "userspace", false},
+			expected:        proxyModeUserspace,
-		{"userspace", "net.experimental.kubernetes.io/proxy-mode", "iptables", false},
+		},
-		{"iptables", "net.experimental.kubernetes.io/proxy-mode", "userspace", true},
+		{ // flag says iptables, version ok
-		{"iptables", "net.experimental.kubernetes.io/proxy-mode", "iptables", true},
+			flag:            "iptables",
-		{"userspace", "net.beta.kubernetes.io/proxy-mode", "userspace", false},
+			iptablesVersion: iptables.MinCheckVersion,
-		{"userspace", "net.beta.kubernetes.io/proxy-mode", "iptables", false},
+			expected:        proxyModeIptables,
-		{"iptables", "net.beta.kubernetes.io/proxy-mode", "userspace", true},
+		},
-		{"iptables", "net.beta.kubernetes.io/proxy-mode", "iptables", true},
+		{ // detect, error
+			flag:          "",
+			iptablesError: fmt.Errorf("oops!"),
+			expected:      proxyModeUserspace,
+		},
+		{ // detect, version too low
+			flag:            "",
+			iptablesVersion: "0.0.0",
+			expected:        proxyModeUserspace,
+		},
+		{ // detect, version ok
+			flag:            "",
+			iptablesVersion: iptables.MinCheckVersion,
+			expected:        proxyModeIptables,
+		},
+		{ // annotation says userspace
+			flag:          "",
+			annotationKey: "net.experimental.kubernetes.io/proxy-mode",
+			annotationVal: "userspace",
+			expected:      proxyModeUserspace,
+		},
+		{ // annotation says iptables, error detecting
+			flag:          "",
+			annotationKey: "net.experimental.kubernetes.io/proxy-mode",
+			annotationVal: "iptables",
+			iptablesError: fmt.Errorf("oops!"),
+			expected:      proxyModeUserspace,
+		},
+		{ // annotation says iptables, version too low
+			flag:            "",
+			annotationKey:   "net.experimental.kubernetes.io/proxy-mode",
+			annotationVal:   "iptables",
+			iptablesVersion: "0.0.0",
+			expected:        proxyModeUserspace,
+		},
+		{ // annotation says iptables, version ok
+			flag:            "",
+			annotationKey:   "net.experimental.kubernetes.io/proxy-mode",
+			annotationVal:   "iptables",
+			iptablesVersion: iptables.MinCheckVersion,
+			expected:        proxyModeIptables,
+		},
+		{ // annotation says something else, version ok
+			flag:            "",
+			annotationKey:   "net.experimental.kubernetes.io/proxy-mode",
+			annotationVal:   "other",
+			iptablesVersion: iptables.MinCheckVersion,
+			expected:        proxyModeIptables,
+		},
+		{ // annotation says nothing, version ok
+			flag:            "",
+			annotationKey:   "net.experimental.kubernetes.io/proxy-mode",
+			annotationVal:   "",
+			iptablesVersion: iptables.MinCheckVersion,
+			expected:        proxyModeIptables,
+		},
+		{ // annotation says userspace
+			flag:          "",
+			annotationKey: "net.beta.kubernetes.io/proxy-mode",
+			annotationVal: "userspace",
+			expected:      proxyModeUserspace,
+		},
+		{ // annotation says iptables, error detecting
+			flag:          "",
+			annotationKey: "net.beta.kubernetes.io/proxy-mode",
+			annotationVal: "iptables",
+			iptablesError: fmt.Errorf("oops!"),
+			expected:      proxyModeUserspace,
+		},
+		{ // annotation says iptables, version too low
+			flag:            "",
+			annotationKey:   "net.beta.kubernetes.io/proxy-mode",
+			annotationVal:   "iptables",
+			iptablesVersion: "0.0.0",
+			expected:        proxyModeUserspace,
+		},
+		{ // annotation says iptables, version ok
+			flag:            "",
+			annotationKey:   "net.beta.kubernetes.io/proxy-mode",
+			annotationVal:   "iptables",
+			iptablesVersion: iptables.MinCheckVersion,
+			expected:        proxyModeIptables,
+		},
+		{ // annotation says something else, version ok
+			flag:            "",
+			annotationKey:   "net.beta.kubernetes.io/proxy-mode",
+			annotationVal:   "other",
+			iptablesVersion: iptables.MinCheckVersion,
+			expected:        proxyModeIptables,
+		},
+		{ // annotation says nothing, version ok
+			flag:            "",
+			annotationKey:   "net.beta.kubernetes.io/proxy-mode",
+			annotationVal:   "",
+			iptablesVersion: iptables.MinCheckVersion,
+			expected:        proxyModeIptables,
+		},
+		{ // flag says userspace, annotation disagrees
+			flag:            "userspace",
+			annotationKey:   "net.experimental.kubernetes.io/proxy-mode",
+			annotationVal:   "iptables",
+			iptablesVersion: iptables.MinCheckVersion,
+			expected:        proxyModeUserspace,
+		},
+		{ // flag says iptables, annotation disagrees
+			flag:            "iptables",
+			annotationKey:   "net.experimental.kubernetes.io/proxy-mode",
+			annotationVal:   "userspace",
+			iptablesVersion: iptables.MinCheckVersion,
+			expected:        proxyModeIptables,
+		},
+		{ // flag says userspace, annotation disagrees
+			flag:            "userspace",
+			annotationKey:   "net.beta.kubernetes.io/proxy-mode",
+			annotationVal:   "iptables",
+			iptablesVersion: iptables.MinCheckVersion,
+			expected:        proxyModeUserspace,
+		},
+		{ // flag says iptables, annotation disagrees
+			flag:            "iptables",
+			annotationKey:   "net.beta.kubernetes.io/proxy-mode",
+			annotationVal:   "userspace",
+			iptablesVersion: iptables.MinCheckVersion,
+			expected:        proxyModeIptables,
+		},
 	}
 	for i, c := range cases {
 		getter := &fakeNodeInterface{}
-		getter.node.Annotations = map[string]string{c.annKey: c.annVal}
+		getter.node.Annotations = map[string]string{c.annotationKey: c.annotationVal}
-		r := mayTryIptablesProxy(c.flag, getter, "host")
+		versioner := &fakeIptablesVersioner{c.iptablesVersion, c.iptablesError}
+		r := getProxyMode(c.flag, getter, "host", versioner)
 		if r != c.expected {
-			t.Errorf("Case[%d] Expected %t, got %t", i, c.expected, r)
+			t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)
 		}
 	}
 }

docs/admin/kube-proxy.md

@@ -68,13 +68,13 @@ kube-proxy
       --masquerade-all[=false]: If using the pure iptables proxy, SNAT everything
       --master="": The address of the Kubernetes API server (overrides any value in kubeconfig)
       --oom-score-adj=-999: The oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000]
-      --proxy-mode="": Which proxy mode to use: 'userspace' (older, stable) or 'iptables' (experimental). If blank, look at the Node object on the Kubernetes API and respect the 'net.experimental.kubernetes.io/proxy-mode' annotation if provided.  Otherwise use the best-available proxy (currently userspace, but may change in future versions).  If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.
+      --proxy-mode="": Which proxy mode to use: 'userspace' (older) or 'iptables' (faster). If blank, look at the Node object on the Kubernetes API and respect the 'net.experimental.kubernetes.io/proxy-mode' annotation if provided.  Otherwise use the best-available proxy (currently iptables).  If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.
       --proxy-port-range=: Range of host ports (beginPort-endPort, inclusive) that may be consumed in order to proxy service traffic. If unspecified (0-0) then ports will be randomly chosen.
       --resource-container="/kube-proxy": Absolute name of the resource-only container to create and run the Kube-proxy in (Default: /kube-proxy).
       --udp-timeout=250ms: How long an idle UDP connection will be kept open (e.g. '250ms', '2s').  Must be greater than 0. Only applicable for proxy-mode=userspace
 ```
 
-###### Auto generated by spf13/cobra on 23-Oct-2015
+###### Auto generated by spf13/cobra on 9-Nov-2015
 
 
 <!-- BEGIN MUNGE: GENERATED_ANALYTICS -->

docs/user-guide/debugging-services.md

@@ -55,6 +55,8 @@ This document will hopefully help you to figure out what's going wrong.
   - [Is the kube-proxy working?](#is-the-kube-proxy-working)
     - [Is kube-proxy running?](#is-kube-proxy-running)
     - [Is kube-proxy writing iptables rules?](#is-kube-proxy-writing-iptables-rules)
+      - [Userspace](#userspace)
+      - [Iptables](#iptables)
     - [Is kube-proxy proxying?](#is-kube-proxy-proxying)
   - [Seek help](#seek-help)
   - [More information](#more-information)
@@ -468,24 +470,16 @@ depends on your `Node` OS.  On some OSes it is a file, such as
 should see something like:
 
 ```console
-I0707 17:34:53.945651   30031 server.go:88] Running in resource-only container "/kube-proxy"
+I1027 22:14:53.995134    5063 server.go:200] Running in resource-only container "/kube-proxy"
-I0707 17:34:53.945921   30031 proxier.go:121] Setting proxy IP to 10.240.115.247 and initializing iptables
+I1027 22:14:53.998163    5063 server.go:247] Using iptables Proxier.
-I0707 17:34:54.053023   30031 roundrobin.go:262] LoadBalancerRR: Setting endpoints for default/kubernetes: to [10.240.169.188:443]
+I1027 22:14:53.999055    5063 server.go:255] Tearing down userspace rules. Errors here are acceptable.
-I0707 17:34:54.053175   30031 roundrobin.go:262] LoadBalancerRR: Setting endpoints for default/hostnames:default to [10.244.0.5:9376 10.244.0.6:9376 10.244.0.7:9376]
+I1027 22:14:54.038140    5063 proxier.go:352] Setting endpoints for "kube-system/kube-dns:dns-tcp" to [10.244.1.3:53]
-I0707 17:34:54.053284   30031 roundrobin.go:262] LoadBalancerRR: Setting endpoints for default/kube-dns:dns to [10.244.3.3:53]
+I1027 22:14:54.038164    5063 proxier.go:352] Setting endpoints for "kube-system/kube-dns:dns" to [10.244.1.3:53]
-I0707 17:34:54.053310   30031 roundrobin.go:262] LoadBalancerRR: Setting endpoints for default/kube-dns:dns-tcp to [10.244.3.3:53]
+I1027 22:14:54.038209    5063 proxier.go:352] Setting endpoints for "default/kubernetes:https" to [10.240.0.2:443]
-I0707 17:34:54.054780   30031 proxier.go:306] Adding new service "default/kubernetes:" at 10.0.0.1:443/TCP
+I1027 22:14:54.038238    5063 proxier.go:429] Not syncing iptables until Services and Endpoints have been received from master
-I0707 17:34:54.054903   30031 proxier.go:247] Proxying for service "default/kubernetes:" on TCP port 40074
+I1027 22:14:54.040048    5063 proxier.go:294] Adding new service "default/kubernetes:https" at 10.0.0.1:443/TCP
-I0707 17:34:54.079181   30031 proxier.go:306] Adding new service "default/hostnames:default" at 10.0.1.175:80/TCP
+I1027 22:14:54.040154    5063 proxier.go:294] Adding new service "kube-system/kube-dns:dns" at 10.0.0.10:53/UDP
-I0707 17:34:54.079273   30031 proxier.go:247] Proxying for service "default/hostnames:default" on TCP port 48577
+I1027 22:14:54.040223    5063 proxier.go:294] Adding new service "kube-system/kube-dns:dns-tcp" at 10.0.0.10:53/TCP
-I0707 17:34:54.113665   30031 proxier.go:306] Adding new service "default/kube-dns:dns" at 10.0.0.10:53/UDP
-I0707 17:34:54.113776   30031 proxier.go:247] Proxying for service "default/kube-dns:dns" on UDP port 34149
-I0707 17:34:54.120224   30031 proxier.go:306] Adding new service "default/kube-dns:dns-tcp" at 10.0.0.10:53/TCP
-I0707 17:34:54.120297   30031 proxier.go:247] Proxying for service "default/kube-dns:dns-tcp" on TCP port 53476
-I0707 17:34:54.902313   30031 proxysocket.go:130] Accepted TCP connection from 10.244.3.3:42670 to 10.244.3.1:40074
-I0707 17:34:54.903107   30031 proxysocket.go:130] Accepted TCP connection from 10.244.3.3:42671 to 10.244.3.1:40074
-I0707 17:35:46.015868   30031 proxysocket.go:246] New UDP connection from 10.244.3.2:57493
-I0707 17:35:46.017061   30031 proxysocket.go:246] New UDP connection from 10.244.3.2:55471
 ```
 
 If you see error messages about not being able to contact the master, you
@@ -497,6 +491,12 @@ One of the main responsibilities of `kube-proxy` is to write the `iptables`
 rules which implement `Service`s.  Let's check that those rules are getting
 written.
 
+The kube-proxy can run in either "userspace" mode or "iptables" mode.
+Hopefully you are using the newer, faster, more stable "iptables" mode.  You
+should see one of the following cases.
+
+#### Userspace
+
 ```console
 u@node$ iptables-save | grep hostnames
 -A KUBE-PORTALS-CONTAINER -d 10.0.1.175/32 -p tcp -m comment --comment "default/hostnames:default" -m tcp --dport 80 -j REDIRECT --to-ports 48577
@@ -508,6 +508,27 @@ example) - a "KUBE-PORTALS-CONTAINER" and a "KUBE-PORTALS-HOST".  If you do
 not see these, try restarting `kube-proxy` with the `-V` flag set to 4, and
 then look at the logs again.
 
+#### Iptables
+
+```console
+u@node$ iptables-save | grep hostnames
+-A KUBE-SEP-57KPRZ3JQVENLNBR -s 10.244.3.6/32 -m comment --comment "default/hostnames:" -j MARK --set-xmark 0x4d415351/0xffffffff
+-A KUBE-SEP-57KPRZ3JQVENLNBR -p tcp -m comment --comment "default/hostnames:" -m tcp -j DNAT --to-destination 10.244.3.6:9376
+-A KUBE-SEP-WNBA2IHDGP2BOBGZ -s 10.244.1.7/32 -m comment --comment "default/hostnames:" -j MARK --set-xmark 0x4d415351/0xffffffff
+-A KUBE-SEP-WNBA2IHDGP2BOBGZ -p tcp -m comment --comment "default/hostnames:" -m tcp -j DNAT --to-destination 10.244.1.7:9376
+-A KUBE-SEP-X3P2623AGDH6CDF3 -s 10.244.2.3/32 -m comment --comment "default/hostnames:" -j MARK --set-xmark 0x4d415351/0xffffffff
+-A KUBE-SEP-X3P2623AGDH6CDF3 -p tcp -m comment --comment "default/hostnames:" -m tcp -j DNAT --to-destination 10.244.2.3:9376
+-A KUBE-SERVICES -d 10.0.1.175/32 -p tcp -m comment --comment "default/hostnames: cluster IP" -m tcp --dport 80 -j KUBE-SVC-NWV5X2332I4OT4T3
+-A KUBE-SVC-NWV5X2332I4OT4T3 -m comment --comment "default/hostnames:" -m statistic --mode random --probability 0.33332999982 -j KUBE-SEP-WNBA2IHDGP2BOBGZ
+-A KUBE-SVC-NWV5X2332I4OT4T3 -m comment --comment "default/hostnames:" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-X3P2623AGDH6CDF3
+-A KUBE-SVC-NWV5X2332I4OT4T3 -m comment --comment "default/hostnames:" -j KUBE-SEP-57KPRZ3JQVENLNBR
+```
+
+There should be 1 rule in `KUBE-SERVICES`, 1 or 2 rules per endpoint in
+`KUBE-SVC-(hash)` (depending on `SessionAffinity`), one `KUBE-SEP-(hash)` chain
+per endpoint, and a few rules in each `KUBE-SEP-(hash)` chain.  The exact rules
+will vary based on your exact config (including node-ports and load-balancers).
+
 ### Is kube-proxy proxying?
 
 Assuming you do see the above rules, try again to access your `Service` by IP:
@@ -517,10 +538,12 @@ u@node$ curl 10.0.1.175:80
 hostnames-0uton
 ```
 
-If this fails, we can try accessing the proxy directly.  Look back at the
+If this fails and you are using the userspace proxy, you can try accessing the
-`iptables-save` output above, and extract the port number that `kube-proxy` is
+proxy directly.  If you are using the iptables proxy, skip this section.
-using for your `Service`.  In the above examples it is "48577".  Now connect to
+
-that:
+Look back at the `iptables-save` output above, and extract the
+port number that `kube-proxy` is using for your `Service`.  In the above
+examples it is "48577".  Now connect to that:
 
 ```console
 u@node$ curl localhost:48577

docs/user-guide/services-iptables-overview.svg

@@ -13,9 +13,9 @@
    height="1052.3622047"
    id="svg2"
    version="1.1"
-   inkscape:version="0.48.3.1 r9886"
+   inkscape:version="0.48.4 r9939"
-   sodipodi:docname="services_detail.svg"
+   sodipodi:docname="services-iptables-overview.svg"
-   inkscape:export-filename="/usr/local/google/home/thockin/src/kubernetes/docs/services_overview.png"
+   inkscape:export-filename="/usr/local/google/home/thockin/src/kubernetes/docs/services-userspace-overview.png"
    inkscape:export-xdpi="76.910004"
    inkscape:export-ydpi="76.910004">
   <defs
@@ -27,16 +27,16 @@
      borderopacity="1.0"
      inkscape:pageopacity="0.0"
      inkscape:pageshadow="2"
-     inkscape:zoom="0.99604166"
+     inkscape:zoom="1.0318369"
-     inkscape:cx="436.19361"
+     inkscape:cx="115.9218"
-     inkscape:cy="503.28586"
+     inkscape:cy="392.30545"
      inkscape:document-units="px"
      inkscape:current-layer="layer1"
      showgrid="false"
-     inkscape:window-width="1228"
+     inkscape:window-width="1552"
-     inkscape:window-height="848"
+     inkscape:window-height="822"
-     inkscape:window-x="364"
+     inkscape:window-x="203"
-     inkscape:window-y="24"
+     inkscape:window-y="50"
      inkscape:window-maximized="0" />
   <metadata
      id="metadata7">
@@ -54,18 +54,26 @@
      inkscape:label="Layer 1"
      inkscape:groupmode="layer"
      id="layer1">
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.842547px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="rect3115"
+       width="545.78632"
+       height="253.1044"
+       x="-43.690273"
+       y="541.54382" />
     <g
-       transform="matrix(1,0,0,-1.1300076,-23.256225,1365.3668)"
+       transform="matrix(1,0,0,-0.92578962,15.303948,1193.1996)"
-       id="g4178-3-98">
+       id="g4178-3-0"
+       style="stroke-width:2.078614;stroke-miterlimit:4;stroke-dasharray:none">
       <path
-         style="fill:none;stroke:#000000;stroke-width:2.82215285;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:none;stroke:#000000;stroke-width:2.078614;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          d="m 337.14286,757.95172 c 0,-71.30383 0,-71.30383 0,-71.30383"
          id="path4174-3-7"
          inkscape:connector-curvature="0" />
       <path
          sodipodi:type="star"
-         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:2.82215309;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:2.078614;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
-         id="path4176-9-8"
+         id="path4176-9-1"
          sodipodi:sides="3"
          sodipodi:cx="308.85715"
          sodipodi:cy="753.79077"
@@ -81,15 +89,46 @@
          inkscape:transform-center-y="-2.5" />
     </g>
     <g
-       id="g4324">
+       transform="matrix(-0.83212211,-1.1231515,0.89570092,-0.89430772,-72.942206,1678.3161)"
+       id="g4178-3-8"
+       style="stroke-width:2.078614;stroke-miterlimit:4;stroke-dasharray:none"
+       inkscape:transform-center-x="84.098741"
+       inkscape:transform-center-y="1.4572787e-05">
       <path
-         style="fill:none;stroke:#000000;stroke-width:2.99999976;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:none;stroke:#000000;stroke-width:2.078614;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         d="m 337.14286,757.95172 c 0,-71.30383 0,-71.30383 0,-71.30383"
+         id="path4174-3-4"
+         inkscape:connector-curvature="0" />
+      <path
+         sodipodi:type="star"
+         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:2.078614;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         id="path4176-9-0"
+         sodipodi:sides="3"
+         sodipodi:cx="308.85715"
+         sodipodi:cy="753.79077"
+         sodipodi:r1="10"
+         sodipodi:r2="5"
+         sodipodi:arg1="2.6179939"
+         sodipodi:arg2="3.6651914"
+         inkscape:flatsided="true"
+         inkscape:rounded="0"
+         inkscape:randomized="0"
+         d="m 300.19689,758.79077 8.66026,-15 8.66025,15 z"
+         transform="translate(28.571429,-62.857143)"
+         inkscape:transform-center-y="-2.5" />
+    </g>
+    <g
+       id="g4324"
+       transform="matrix(0.96592583,0.25881905,0.25881905,-0.96592583,-272.81074,1126.238)"
+       style="stroke-width:5;stroke-miterlimit:4;stroke-dasharray:none">
+      <path
+         style="fill:none;stroke:#000000;stroke-width:5;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          d="M 340.43856,497.06486 C 238.47092,383.2788 238.47092,383.2788 238.47092,383.2788"
          id="path4174-3-2"
          inkscape:connector-curvature="0" />
       <path
          sodipodi:type="star"
-         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:2.82215309;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:4.70358849;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          id="path4176-9-9"
          sodipodi:sides="3"
          sodipodi:cx="308.85715"
@@ -106,16 +145,17 @@
          inkscape:transform-center-y="-2.5" />
     </g>
     <g
-       transform="matrix(-1,0,0,1,718.68427,0.32076964)"
+       transform="matrix(-0.96592583,0.25881905,-0.25881905,-0.96592583,654.32964,1126.238)"
-       id="g4324-8">
+       id="g4324-8"
+       style="stroke-width:5;stroke-miterlimit:4;stroke-dasharray:none">
       <path
-         style="fill:none;stroke:#000000;stroke-width:2.99999976;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:none;stroke:#000000;stroke-width:5;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          d="M 340.43856,497.06486 C 238.47092,383.2788 238.47092,383.2788 238.47092,383.2788"
          id="path4174-3-2-7"
          inkscape:connector-curvature="0" />
       <path
          sodipodi:type="star"
-         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:2.82215309;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:4.70358849;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          id="path4176-9-9-3"
          sodipodi:sides="3"
          sodipodi:cx="308.85715"
@@ -132,16 +172,17 @@
          inkscape:transform-center-y="-2.5" />
     </g>
     <g
-       transform="matrix(1,0,0,1.3566066,10.430689,-549.99231)"
+       transform="matrix(1,0,0,-1.3566066,-154.75999,1749.5431)"
-       id="g4178-3-9">
+       id="g4178-3-9"
+       style="stroke-width:4.29282379;stroke-miterlimit:4;stroke-dasharray:none">
       <path
-         style="fill:none;stroke:#000000;stroke-width:2.57569385;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:none;stroke:#000000;stroke-width:4.29282379;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          d="m 337.14286,757.95172 c 0,-71.30383 0,-71.30383 0,-71.30383"
          id="path4174-3-8"
          inkscape:connector-curvature="0" />
       <path
          sodipodi:type="star"
-         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:2.57569408;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:4.29282379;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          id="path4176-9-5"
          sodipodi:sides="3"
          sodipodi:cx="308.85715"
@@ -157,37 +198,11 @@
          transform="translate(28.571429,-62.857143)"
          inkscape:transform-center-y="-2.5" />
     </g>
-    <g
-       transform="matrix(1,0,0,0.83995083,5.8686441,145.11325)"
-       id="g4178">
-      <path
-         style="fill:none;stroke:#000000;stroke-width:3.27336383;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
-         d="m 337.14286,757.95172 c 0,-71.30383 0,-71.30383 0,-71.30383"
-         id="path4174"
-         inkscape:connector-curvature="0" />
-      <path
-         sodipodi:type="star"
-         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:3.27336407;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
-         id="path4176"
-         sodipodi:sides="3"
-         sodipodi:cx="308.85715"
-         sodipodi:cy="753.79077"
-         sodipodi:r1="10"
-         sodipodi:r2="5"
-         sodipodi:arg1="2.6179939"
-         sodipodi:arg2="3.6651914"
-         inkscape:flatsided="true"
-         inkscape:rounded="0"
-         inkscape:randomized="0"
-         d="m 300.19689,758.79077 8.66026,-15 8.66025,15 z"
-         transform="translate(28.571429,-62.857143)"
-         inkscape:transform-center-y="-2.5" />
-    </g>
     <g
        id="g3937"
-       transform="translate(-27.782873,191.54649)">
+       transform="translate(-212.35646,769.73074)">
       <g
-         transform="translate(0,6.5250001e-6)"
+         transform="matrix(0.88792337,0,0,1,43.50975,6.5250001e-6)"
          id="g3868">
         <rect
            style="fill:#85bff1;fill-opacity:1;stroke:#000000;stroke-width:2;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
@@ -233,7 +248,7 @@
       </g>
       <g
          id="g3868-7"
-         transform="translate(246.07142,6.5250001e-6)">
+         transform="matrix(0.88792337,0,0,1,262.00231,6.5250001e-6)">
         <rect
            style="fill:#85bff1;fill-opacity:1;stroke:#000000;stroke-width:2;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
            id="rect2985-1"
@@ -278,7 +293,7 @@
       </g>
       <g
          id="g3868-3"
-         transform="translate(492.14285,6.5250001e-6)">
+         transform="matrix(0.88792337,0,0,1,480.49489,6.5250001e-6)">
         <rect
            style="fill:#85bff1;fill-opacity:1;stroke:#000000;stroke-width:2;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
            id="rect2985-2"
@@ -323,42 +338,17 @@
       </g>
     </g>
     <g
-       transform="matrix(-0.5569815,0.8305249,-0.93849945,-0.62939332,1043.1434,624.89979)"
+       transform="matrix(0.91192623,-0.41035418,-0.37990164,-0.84425184,113.60453,1385.4009)"
-       id="g4178-3-4">
+       id="g4178-3"
+       style="stroke-width:5.19653511;stroke-miterlimit:4;stroke-dasharray:none">
       <path
-         style="fill:none;stroke:#000000;stroke-width:2.82215285;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:none;stroke:#000000;stroke-width:5.19653511;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
-         d="m 337.14286,757.95172 c 0,-71.30383 0,-71.30383 0,-71.30383"
-         id="path4174-3-9"
-         inkscape:connector-curvature="0" />
-      <path
-         sodipodi:type="star"
-         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:2.82215309;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
-         id="path4176-9-1"
-         sodipodi:sides="3"
-         sodipodi:cx="308.85715"
-         sodipodi:cy="753.79077"
-         sodipodi:r1="10"
-         sodipodi:r2="5"
-         sodipodi:arg1="2.6179939"
-         sodipodi:arg2="3.6651914"
-         inkscape:flatsided="true"
-         inkscape:rounded="0"
-         inkscape:randomized="0"
-         d="m 300.19689,758.79077 8.66026,-15 8.66025,15 z"
-         transform="translate(28.571429,-62.857143)"
-         inkscape:transform-center-y="-2.5" />
-    </g>
-    <g
-       transform="matrix(1,0,0,1.1300076,19.868644,-230.41621)"
-       id="g4178-3">
-      <path
-         style="fill:none;stroke:#000000;stroke-width:2.82215285;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          d="m 337.14286,757.95172 c 0,-71.30383 0,-71.30383 0,-71.30383"
          id="path4174-3"
          inkscape:connector-curvature="0" />
       <path
          sodipodi:type="star"
-         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:2.82215309;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:5.19653511;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          id="path4176-9"
          sodipodi:sides="3"
          sodipodi:cx="308.85715"
@@ -375,7 +365,7 @@
          inkscape:transform-center-y="-2.5" />
     </g>
     <g
-       transform="translate(9.4642913,66)"
+       transform="matrix(0.89067003,0,0,1,-194.97295,-142.36286)"
        id="g4090">
       <rect
          y="704.50507"
@@ -385,47 +375,26 @@
          id="rect2985-4"
          style="fill:#f1cb85;fill-opacity:1;stroke:#000000;stroke-width:2;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" />
       <g
-         transform="translate(249.2817,652.74516)"
+         transform="translate(217.6177,652.82516)"
          id="g3861-6">
         <text
            xml:space="preserve"
            style="font-size:40px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Ubuntu Mono;-inkscape-font-specification:Ubuntu Mono"
-           x="36.710861"
+           x="67.574867"
-           y="91.845612"
+           y="91.765617"
            id="text3755-32"
            sodipodi:linespacing="125%"
            inkscape:transform-center-x="-70"
            inkscape:transform-center-y="-11.264"><tspan
              sodipodi:role="line"
              id="tspan3757-9"
-             x="36.710861"
+             x="67.574867"
-             y="91.845612"
+             y="91.765617"
-             style="font-size:32px;text-align:start;text-anchor:start">Client</tspan></text>
+             style="font-size:32px;text-align:start;text-anchor:start">Client </tspan></text>
       </g>
     </g>
     <g
-       transform="translate(24.285715,159.42857)"
+       transform="matrix(0.89067003,0,0,1,199.65922,-24.251798)"
-       id="g4114">
-      <path
-         inkscape:connector-curvature="0"
-         style="fill:#ededed;fill-opacity:1;stroke:#000000;stroke-width:1.99999988;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
-         d="m 282.87054,438.5755 c -23.66935,0 -42.875,11.54365 -42.875,25.78345 0,1.69709 0.29232,3.36317 0.8125,4.96869 -5.77989,-1.60822 -12.0611,-2.49777 -18.65625,-2.49777 -28.00873,0 -50.71875,15.92203 -50.71875,35.58653 0,19.66449 22.71002,35.61339 50.71875,35.61339 9.72296,0 18.78316,-1.93319 26.5,-5.26412 10.70208,13.21239 35.10628,22.45308 63.5,22.45308 23.13948,0 43.60406,-6.13049 56.1875,-15.55064 12.16376,6.53313 29.85326,10.63567 49.53125,10.63567 36.68749,0 66.40625,-14.27678 66.40625,-31.90702 0,-17.63023 -29.71876,-31.93387 -66.40625,-31.93387 -0.61492,0 -1.23284,0.0189 -1.84375,0.0268 0.72778,-1.79609 1.125,-3.66107 1.125,-5.55955 0,-15.93503 -26.86291,-28.84524 -60,-28.84524 -12.3074,0 -23.75966,1.77775 -33.28125,4.8344 -5.31552,-10.60488 -21.63938,-18.34385 -41,-18.34385 z"
-         id="path4096" />
-      <text
-         xml:space="preserve"
-         style="font-size:40px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Ubuntu Mono;-inkscape-font-specification:Ubuntu Mono"
-         x="270.39322"
-         y="507.15195"
-         id="text4108"
-         sodipodi:linespacing="125%"><tspan
-           sodipodi:role="line"
-           x="270.39322"
-           y="507.15195"
-           id="tspan4112"
-           style="font-size:22px">iptables</tspan></text>
-    </g>
-    <g
-       transform="translate(167.67856,-111.42858)"
        id="g4168">
       <rect
          y="588.79077"
@@ -435,26 +404,26 @@
          id="rect2985-4-0"
          style="fill:#b9f185;fill-opacity:1;stroke:#000000;stroke-width:2;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" />
       <g
-         transform="translate(58.491433,534.63087)"
+         transform="translate(34.747433,534.26287)"
          id="g3861-6-2">
         <text
            xml:space="preserve"
            style="font-size:40px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Ubuntu Mono;-inkscape-font-specification:Ubuntu Mono"
-           x="36.710861"
+           x="60.454861"
-           y="91.845612"
+           y="92.213608"
            id="text3755-32-8"
            sodipodi:linespacing="125%"
            inkscape:transform-center-x="-70"
            inkscape:transform-center-y="-11.264"><tspan
              sodipodi:role="line"
              id="tspan3757-9-4"
-             x="36.710861"
+             x="60.454861"
-             y="91.845612"
+             y="92.213608"
              style="font-size:32px;text-align:start;text-anchor:start">kube-proxy</tspan></text>
       </g>
     </g>
     <g
-       transform="translate(-102.23193,-119.15421)"
+       transform="translate(188.04333,-29.041887)"
        id="g4168-5">
       <g
          transform="translate(22.087429,-86.34177)"
@@ -486,85 +455,44 @@
         </g>
       </g>
     </g>
+    <path
+       style="fill:#ffe680;fill-opacity:1;fill-rule:nonzero;stroke:#000000;stroke-width:1.77870166;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+       d="m 184.45745,671.01905 c -14.46832,0 -26.98388,8.64301 -32.95619,21.20091 -5.04913,-0.90778 -10.34543,-1.41525 -15.81006,-1.41525 -34.32572,0 -62.154694,19.09673 -62.154694,42.62382 0,23.52709 27.828974,42.59606 62.154694,42.59606 16.91161,0 32.24392,-4.64352 43.44984,-12.15444 7.36101,16.27536 34.00477,28.33262 65.74535,28.33262 33.5174,0 61.35664,-13.44827 66.80308,-31.10761 17.02922,-5.30597 28.58615,-15.7069 28.58615,-27.66663 0,-17.34826 -24.35383,-31.41286 -54.38884,-31.41286 -8.45761,0 -16.46469,1.0906 -23.60375,3.08023 -2.04151,-10.49178 -14.83542,-18.59242 -30.33973,-18.59242 -5.80798,0 -11.23748,1.16534 -15.86573,3.13574 -6.38532,-11.14574 -18.16606,-18.62017 -31.62012,-18.62017 z"
+       id="path3884"
+       inkscape:connector-curvature="0" />
+    <g
+       transform="matrix(0.89067003,0,0,1,34.810052,629.61733)"
+       id="g3861-6-28">
+      <text
+         xml:space="preserve"
+         style="font-size:40px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Ubuntu Mono;-inkscape-font-specification:Ubuntu Mono"
+         x="93.689468"
+         y="89.827324"
+         id="text3755-32-4"
+         sodipodi:linespacing="125%"
+         inkscape:transform-center-x="-70"
+         inkscape:transform-center-y="-11.264"><tspan
+           sodipodi:role="line"
+           id="tspan3757-9-5"
+           x="93.689468"
+           y="89.827324"
+           style="font-size:32px;text-align:start;text-anchor:start">ServiceIP</tspan><tspan
+           sodipodi:role="line"
+           x="93.689468"
+           y="129.82733"
+           style="font-size:32px;text-align:start;text-anchor:start"
+           id="tspan3919">(iptables) </tspan></text>
+    </g>
     <text
        xml:space="preserve"
-       style="font-size:40px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Ubuntu Mono;-inkscape-font-specification:Ubuntu Mono"
+       style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
-       x="354.03052"
+       x="389.59647"
-       y="752.17395"
+       y="786.81635"
-       id="text4777"
+       id="text3885"
        sodipodi:linespacing="125%"><tspan
          sodipodi:role="line"
-         id="tspan4779"
+         id="tspan3887"
-         x="354.03052"
+         x="389.59647"
-         y="752.17395"
+         y="786.81635">Node</tspan></text>
-         style="font-size:22px">3) connect to 10.0.0.1:1234</tspan></text>
-    <text
-       xml:space="preserve"
-       style="font-size:40px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Ubuntu Mono;-inkscape-font-specification:Ubuntu Mono"
-       x="381.81412"
-       y="563.21899"
-       id="text4777-1"
-       sodipodi:linespacing="125%"><tspan
-         sodipodi:role="line"
-         x="381.81412"
-         y="563.21899"
-         style="font-size:22px"
-         id="tspan4804">4) redirect to (random)</tspan><tspan
-         sodipodi:role="line"
-         x="381.81412"
-         y="590.71899"
-         style="font-size:22px"
-         id="tspan3060">proxy port</tspan></text>
-    <text
-       xml:space="preserve"
-       style="font-size:40px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Ubuntu Mono;-inkscape-font-specification:Ubuntu Mono"
-       x="-11.495128"
-       y="476.92422"
-       id="text4777-1-3"
-       sodipodi:linespacing="125%"><tspan
-         sodipodi:role="line"
-         x="-11.495128"
-         y="476.92422"
-         style="font-size:22px"
-         id="tspan4804-8">1) watch Services </tspan><tspan
-         sodipodi:role="line"
-         x="-11.495128"
-         y="504.42422"
-         style="font-size:22px"
-         id="tspan3056">and Endpoints</tspan></text>
-    <text
-       xml:space="preserve"
-       style="font-size:40px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Ubuntu Mono;-inkscape-font-specification:Ubuntu Mono"
-       x="53.554245"
-       y="557.18707"
-       id="text4777-1-3-5"
-       sodipodi:linespacing="125%"><tspan
-         sodipodi:role="line"
-         x="53.554245"
-         y="557.18707"
-         style="font-size:22px"
-         id="tspan4804-8-5">2) open proxy port </tspan><tspan
-         sodipodi:role="line"
-         x="53.554245"
-         y="584.68707"
-         style="font-size:22px"
-         id="tspan3058">and set portal rules</tspan></text>
-    <text
-       xml:space="preserve"
-       style="font-size:40px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Ubuntu Mono;-inkscape-font-specification:Ubuntu Mono"
-       x="450.63913"
-       y="442.09073"
-       id="text4777-1-2"
-       sodipodi:linespacing="125%"><tspan
-         sodipodi:role="line"
-         x="450.63913"
-         y="442.09073"
-         style="font-size:22px"
-         id="tspan4804-9">5) proxy to a backend</tspan><tspan
-         sodipodi:role="line"
-         x="450.63913"
-         y="469.59073"
-         style="font-size:22px"
-         id="tspan3060-8" /></text>
   </g>
 </svg>

docs/user-guide/services-userspace-overview.svg

@@ -13,8 +13,8 @@
    height="1052.3622047"
    id="svg2"
    version="1.1"
-   inkscape:version="0.48.3.1 r9886"
+   inkscape:version="0.48.4 r9939"
-   sodipodi:docname="services_overview.svg"
+   sodipodi:docname="services-userspace-overview.svg"
    inkscape:export-filename="/usr/local/google/home/thockin/src/kubernetes/docs/services_overview.png"
    inkscape:export-xdpi="76.910004"
    inkscape:export-ydpi="76.910004">
@@ -28,15 +28,15 @@
      inkscape:pageopacity="0.0"
      inkscape:pageshadow="2"
      inkscape:zoom="1.0318369"
-     inkscape:cx="351.19865"
+     inkscape:cx="291.9254"
-     inkscape:cy="624.90035"
+     inkscape:cy="392.30545"
      inkscape:document-units="px"
-     inkscape:current-layer="g4090"
+     inkscape:current-layer="layer1"
      showgrid="false"
-     inkscape:window-width="1228"
+     inkscape:window-width="1552"
-     inkscape:window-height="848"
+     inkscape:window-height="822"
-     inkscape:window-x="364"
+     inkscape:window-x="46"
-     inkscape:window-y="24"
+     inkscape:window-y="47"
      inkscape:window-maximized="0" />
   <metadata
      id="metadata7">
@@ -55,15 +55,44 @@
      inkscape:groupmode="layer"
      id="layer1">
     <g
-       id="g4324">
+       transform="matrix(0,-1,-0.92578962,0,936.44413,1029.2686)"
+       id="g4178-3-8"
+       style="stroke-width:5.19653493;stroke-miterlimit:4;stroke-dasharray:none">
       <path
-         style="fill:none;stroke:#000000;stroke-width:2.99999976;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:none;stroke:#000000;stroke-width:5.19653493;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         d="m 337.14286,757.95172 c 0,-71.30383 0,-71.30383 0,-71.30383"
+         id="path4174-3-4"
+         inkscape:connector-curvature="0" />
+      <path
+         sodipodi:type="star"
+         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:5.19653493;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         id="path4176-9-0"
+         sodipodi:sides="3"
+         sodipodi:cx="308.85715"
+         sodipodi:cy="753.79077"
+         sodipodi:r1="10"
+         sodipodi:r2="5"
+         sodipodi:arg1="2.6179939"
+         sodipodi:arg2="3.6651914"
+         inkscape:flatsided="true"
+         inkscape:rounded="0"
+         inkscape:randomized="0"
+         d="m 300.19689,758.79077 8.66026,-15 8.66025,15 z"
+         transform="translate(28.571429,-62.857143)"
+         inkscape:transform-center-y="-2.5" />
+    </g>
+    <g
+       id="g4324"
+       transform="matrix(0.96592583,0.25881905,0.25881905,-0.96592583,-38.810744,1076.238)"
+       style="stroke-width:4.99999998;stroke-miterlimit:4;stroke-dasharray:none">
+      <path
+         style="fill:none;stroke:#000000;stroke-width:4.99999998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          d="M 340.43856,497.06486 C 238.47092,383.2788 238.47092,383.2788 238.47092,383.2788"
          id="path4174-3-2"
          inkscape:connector-curvature="0" />
       <path
          sodipodi:type="star"
-         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:2.82215309;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:4.70358849;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          id="path4176-9-9"
          sodipodi:sides="3"
          sodipodi:cx="308.85715"
@@ -80,16 +109,17 @@
          inkscape:transform-center-y="-2.5" />
     </g>
     <g
-       transform="matrix(-1,0,0,1,718.68427,0.32076964)"
+       transform="matrix(-0.96592583,0.25881905,-0.25881905,-0.96592583,888.32964,1076.238)"
-       id="g4324-8">
+       id="g4324-8"
+       style="stroke-width:4.99999998;stroke-miterlimit:4;stroke-dasharray:none">
       <path
-         style="fill:none;stroke:#000000;stroke-width:2.99999976;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:none;stroke:#000000;stroke-width:4.99999998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          d="M 340.43856,497.06486 C 238.47092,383.2788 238.47092,383.2788 238.47092,383.2788"
          id="path4174-3-2-7"
          inkscape:connector-curvature="0" />
       <path
          sodipodi:type="star"
-         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:2.82215309;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:4.70358849;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          id="path4176-9-9-3"
          sodipodi:sides="3"
          sodipodi:cx="308.85715"
@@ -106,16 +136,17 @@
          inkscape:transform-center-y="-2.5" />
     </g>
     <g
-       transform="matrix(1,0,0,1.3566066,10.430689,-549.99231)"
+       transform="matrix(1,0,0,-1.3566066,79.240014,1699.5431)"
-       id="g4178-3-9">
+       id="g4178-3-9"
+       style="stroke-width:4.29282359;stroke-miterlimit:4;stroke-dasharray:none">
       <path
-         style="fill:none;stroke:#000000;stroke-width:2.57569385;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:none;stroke:#000000;stroke-width:4.29282359;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          d="m 337.14286,757.95172 c 0,-71.30383 0,-71.30383 0,-71.30383"
          id="path4174-3-8"
          inkscape:connector-curvature="0" />
       <path
          sodipodi:type="star"
-         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:2.57569408;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:4.29282359;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          id="path4176-9-5"
          sodipodi:sides="3"
          sodipodi:cx="308.85715"
@@ -133,9 +164,9 @@
     </g>
     <g
        id="g3937"
-       transform="translate(-27.782873,191.54649)">
+       transform="translate(21.643544,719.73074)">
       <g
-         transform="translate(0,6.5250001e-6)"
+         transform="matrix(0.88792337,0,0,1,43.50975,6.5250001e-6)"
          id="g3868">
         <rect
            style="fill:#85bff1;fill-opacity:1;stroke:#000000;stroke-width:2;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
@@ -181,7 +212,7 @@
       </g>
       <g
          id="g3868-7"
-         transform="translate(246.07142,6.5250001e-6)">
+         transform="matrix(0.88792337,0,0,1,262.00231,6.5250001e-6)">
         <rect
            style="fill:#85bff1;fill-opacity:1;stroke:#000000;stroke-width:2;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
            id="rect2985-1"
@@ -226,7 +257,7 @@
       </g>
       <g
          id="g3868-3"
-         transform="translate(492.14285,6.5250001e-6)">
+         transform="matrix(0.88792337,0,0,1,480.49489,6.5250001e-6)">
         <rect
            style="fill:#85bff1;fill-opacity:1;stroke:#000000;stroke-width:2;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
            id="rect2985-2"
@@ -271,16 +302,17 @@
       </g>
     </g>
     <g
-       transform="matrix(-0.5569815,0.8305249,-0.93849945,-0.62939332,1043.1434,624.89979)"
+       transform="matrix(-0.8305249,-0.5569815,0.62939332,-0.93849945,365.54855,1487.8396)"
-       id="g4178-3-4">
+       id="g4178-3-4"
+       style="stroke-width:1.88143539;stroke-miterlimit:4;stroke-dasharray:none">
       <path
-         style="fill:none;stroke:#000000;stroke-width:2.82215285;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:none;stroke:#000000;stroke-width:1.88143539;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          d="m 337.14286,757.95172 c 0,-71.30383 0,-71.30383 0,-71.30383"
          id="path4174-3-9"
          inkscape:connector-curvature="0" />
       <path
          sodipodi:type="star"
-         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:2.82215309;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:1.88143539;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          id="path4176-9-1"
          sodipodi:sides="3"
          sodipodi:cx="308.85715"
@@ -297,16 +329,17 @@
          inkscape:transform-center-y="-2.5" />
     </g>
     <g
-       transform="matrix(1,0,0,1.1300076,5.8686441,-230.41621)"
+       transform="matrix(1,0,0,-0.92578962,-170.98136,1268.7699)"
-       id="g4178-3">
+       id="g4178-3"
+       style="stroke-width:5.19653511;stroke-miterlimit:4;stroke-dasharray:none">
       <path
-         style="fill:none;stroke:#000000;stroke-width:2.82215285;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:none;stroke:#000000;stroke-width:5.19653511;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          d="m 337.14286,757.95172 c 0,-71.30383 0,-71.30383 0,-71.30383"
          id="path4174-3"
          inkscape:connector-curvature="0" />
       <path
          sodipodi:type="star"
-         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:2.82215309;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:5.19653511;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
          id="path4176-9"
          sodipodi:sides="3"
          sodipodi:cx="308.85715"
@@ -323,7 +356,7 @@
          inkscape:transform-center-y="-2.5" />
     </g>
     <g
-       transform="translate(11.472239,-104.6279)"
+       transform="matrix(0.89067003,0,0,1,-130.97295,-172.36286)"
        id="g4090">
       <rect
          y="704.50507"
@@ -352,7 +385,7 @@
       </g>
     </g>
     <g
-       transform="translate(167.67856,-111.42858)"
+       transform="matrix(0.89067003,0,0,1,263.65922,74.205473)"
        id="g4168">
       <rect
          y="588.79077"
@@ -381,7 +414,7 @@
       </g>
     </g>
     <g
-       transform="translate(-102.23193,-119.15421)"
+       transform="translate(478.82336,27.291965)"
        id="g4168-5">
       <g
          transform="translate(22.087429,-86.34177)"
@@ -413,5 +446,51 @@
         </g>
       </g>
     </g>
+    <path
+       style="fill:#ffe680;fill-opacity:1;fill-rule:nonzero;stroke:#000000;stroke-width:1.77870166;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"
+       d="m 118.55556,629.08076 c -14.46832,0 -26.983883,8.64301 -32.956193,21.20091 -5.04913,-0.90778 -10.34543,-1.41525 -15.81006,-1.41525 -34.32572,0 -62.1546887,19.09673 -62.1546887,42.62382 0,23.52709 27.8289687,42.59606 62.1546887,42.59606 16.91161,0 32.243923,-4.64352 43.449843,-12.15444 7.36101,16.27536 34.00477,28.33262 65.74535,28.33262 33.5174,0 61.35664,-13.44827 66.80308,-31.10761 17.02922,-5.30597 28.58615,-15.7069 28.58615,-27.66663 0,-17.34826 -24.35383,-31.41286 -54.38884,-31.41286 -8.45761,0 -16.46469,1.0906 -23.60375,3.08023 -2.04151,-10.49178 -14.83542,-18.59242 -30.33973,-18.59242 -5.80798,0 -11.23748,1.16534 -15.86573,3.13574 -6.38532,-11.14574 -18.16606,-18.62017 -31.62012,-18.62017 z"
+       id="path3884"
+       inkscape:connector-curvature="0" />
+    <g
+       transform="matrix(0.89067003,0,0,1,-31.091836,587.67904)"
+       id="g3861-6-28">
+      <text
+         xml:space="preserve"
+         style="font-size:40px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Ubuntu Mono;-inkscape-font-specification:Ubuntu Mono"
+         x="93.689468"
+         y="89.827324"
+         id="text3755-32-4"
+         sodipodi:linespacing="125%"
+         inkscape:transform-center-x="-70"
+         inkscape:transform-center-y="-11.264"><tspan
+           sodipodi:role="line"
+           id="tspan3757-9-5"
+           x="93.689468"
+           y="89.827324"
+           style="font-size:32px;text-align:start;text-anchor:start">ServiceIP</tspan><tspan
+           sodipodi:role="line"
+           x="93.689468"
+           y="129.82733"
+           style="font-size:32px;text-align:start;text-anchor:start"
+           id="tspan3919">(iptables) </tspan></text>
+    </g>
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.92393565px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="rect3889"
+       width="544.73572"
+       height="267.56021"
+       x="-3.9146113"
+       y="484.40494" />
+    <text
+       xml:space="preserve"
+       style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"
+       x="0.969145"
+       y="521.27051"
+       id="text4399"
+       sodipodi:linespacing="125%"><tspan
+         sodipodi:role="line"
+         id="tspan4401"
+         x="0.969145"
+         y="521.27051">Node</tspan></text>
   </g>
 </svg>

docs/user-guide/services.md

@@ -41,6 +41,8 @@ Documentation for other releases can be found at
   - [Defining a service](#defining-a-service)
     - [Services without selectors](#services-without-selectors)
   - [Virtual IPs and service proxies](#virtual-ips-and-service-proxies)
+    - [Proxy-mode: userspace](#proxy-mode-userspace)
+    - [Proxy-mode: iptables](#proxy-mode-iptables)
   - [Multi-Port Services](#multi-port-services)
   - [Choosing your own IP address](#choosing-your-own-ip-address)
     - [Why not use round-robin DNS?](#why-not-use-round-robin-dns)
@@ -57,6 +59,8 @@ Documentation for other releases can be found at
   - [The gory details of virtual IPs](#the-gory-details-of-virtual-ips)
     - [Avoiding collisions](#avoiding-collisions)
     - [IPs and VIPs](#ips-and-vips)
+      - [Userspace](#userspace)
+      - [Iptables](#iptables)
   - [API Object](#api-object)
 
 <!-- END MUNGE: GENERATED_TOC -->
@@ -206,26 +210,55 @@ this example).
 ## Virtual IPs and service proxies
 
 Every node in a Kubernetes cluster runs a `kube-proxy`.  This application
-watches the Kubernetes master for the addition and removal of `Service`
+is responsible for implementing a form of virtual IP for `Service`s.  In
-and `Endpoints` objects. For each `Service` it opens a port (randomly chosen)
+Kubernetes v1.0 the proxy was purely in userspace.  In Kubernetes v1.1 an
-on the local node.  Any connections to `service`  port will be proxied to one of
+iptables proxy was added, but was not the default operating mode.  In
-the corresponding backend `Pods`.  Which backend `Pod`  to use is decided based on the
+Kubernetes v1.2 we expect the iptables proxy to be the default.
+
+As of Kubernetes v1.0, `Services` are a "layer 3" (TCP/UDP over IP) construct.
+In Kubernetes v1.1 the `Ingress` API was added (beta) to represent "layer 7"
+(HTTP) services.
+
+### Proxy-mode: userspace
+
+In this mode, kube-proxy watches the Kubernetes master for the addition and
+removal of `Service` and `Endpoints` objects. For each `Service` it opens a
+port (randomly chosen) on the local node.  Any connections to this "proxy port"
+will be proxied to one of the `Service`'s backend `Pods` (as reported in
+`Endpoints`).  Which backend `Pod`  to use is decided based on the
 `SessionAffinity` of the `Service`.  Lastly, it installs iptables rules which
-capture traffic to the `Service`'s cluster IP (which is virtual) and `Port` then
+capture traffic to the `Service`'s `clusterIP` (which is virtual) and `Port`
-redirects that traffic to the backend `Pod` (`Endpoints`).
+and redirects that traffic to the proxy port which proxies the a backend `Pod`.
 
-The net result is that any traffic bound for the `Service` is proxied to an
+The net result is that any traffic bound for the `Service`'s IP:Port is proxied
-appropriate backend without the clients knowing anything about Kubernetes or
+to an appropriate backend without the clients knowing anything about Kubernetes
-`Services` or `Pods`.
+or `Services` or `Pods`.
-
-![Services overview diagram](services-overview.png)
 
 By default, the choice of backend is round robin.  Client-IP based session affinity
 can be selected by setting `service.spec.sessionAffinity` to `"ClientIP"` (the
 default is `"None"`).
 
-As of Kubernetes 1.0, `Services` are a "layer 3" (TCP/UDP over IP) construct.  We do not
+![Services overview diagram for userspace proxy](services-userspace-overview.png)
-yet have a concept of "layer 7" (HTTP) services.
+
+### Proxy-mode: iptables
+
+In this mode, kube-proxy watches the Kubernetes master for the addition and
+removal of `Service` and `Endpoints` objects. For each `Service` it installs
+iptables rules which capture traffic to the `Service`'s `clusterIP` (which is
+virtual) and `Port` and redirects that traffic to one of the `Service`'s
+backend sets.  For each `Endpoints` object it installs iptables rules which
+select a backend `Pod`.
+
+By default, the choice of backend is random.  Client-IP based session affinity
+can be selected by setting `service.spec.sessionAffinity` to `"ClientIP"` (the
+default is `"None"`).
+
+As with the userspace proxy, the net result is that any traffic bound for the
+`Service`'s IP:Port is proxied to an appropriate backend without the clients
+knowing anything about Kubernetes or `Services` or `Pods`. This should be
+faster and more reliable than the userspace proxy.
+
+![Services overview diagram for iptables proxy](services-iptables-overview.png)
 
 ## Multi-Port Services
 
@@ -494,14 +527,14 @@ In the example below, my-service can be accessed by clients on 80.11.12.10:80 (e
 
 ## Shortcomings
 
-We expect that using iptables and userspace proxies for VIPs will work at
+Using the userspace proxy for VIPs will work at small to medium scale, but will
-small to medium scale, but may not scale to very large clusters with thousands
+not scale to very large clusters with thousands of Services.  See [the original
-of Services.  See [the original design proposal for
+design proposal for portals](http://issue.k8s.io/1107) for more details.
-portals](http://issue.k8s.io/1107) for more
-details.
 
-Using the kube-proxy obscures the source-IP of a packet accessing a `Service`.
+Using the userspace proxy obscures the source-IP of a packet accessing a `Service`.
-This makes some kinds of firewalling impossible.
+This makes some kinds of firewalling impossible.  The iptables proxier does not
+obscure in-cluster source IPs, but it does still impact clients coming through
+a load-balancer or node-port.
 
 LoadBalancers only support TCP, not UDP.
 
@@ -517,13 +550,7 @@ simple round robin balancing, for example master-elected or sharded.  We also
 envision that some `Services` will have "real" load balancers, in which case the
 VIP will simply transport the packets there.
 
-There's a
+We intend to improve our support for L7 (HTTP) `Services`.
-[proposal](http://issue.k8s.io/3760) to
-eliminate userspace proxying in favor of doing it all in iptables.  This should
-perform better and fix the source-IP obfuscation, though is less flexible than
-arbitrary userspace code.
-
-We intend to have first-class support for L7 (HTTP) `Services`.
 
 We intend to have more flexible ingress modes for `Services` which encompass
 the current `ClusterIP`, `NodePort`, and `LoadBalancer` modes and more.
@@ -565,6 +592,11 @@ VIP, their traffic is automatically transported to an appropriate endpoint.
 The environment variables and DNS for `Services` are actually populated in
 terms of the `Service`'s VIP and port.
 
+We support two proxy modes - userspace and iptables, which operate slightly
+differently.
+
+#### Userspace
+
 As an example, consider the image processing application described above.
 When the backend `Service` is created, the Kubernetes master assigns a virtual
 IP address, for example 10.0.0.1.  Assuming the `Service` port is 1234, the
@@ -581,7 +613,24 @@ This means that `Service` owners can choose any port they want without risk of
 collision.  Clients can simply connect to an IP and port, without being aware
 of which `Pods` they are actually accessing.
 
-![Services detailed diagram](services-detail.png)
+#### Iptables
+
+Again, consider the image processing application described above.
+When the backend `Service` is created, the Kubernetes master assigns a virtual
+IP address, for example 10.0.0.1.  Assuming the `Service` port is 1234, the
+`Service` is observed by all of the `kube-proxy` instances in the cluster.
+When a proxy sees a new `Service`, it installs a series of iptables rules which
+redirect from the VIP to per-`Service` rules.  The per-`Service` rules link to
+per-`Endpoint` rules which redirect (Destination NAT) to the backends.
+
+When a client connects to the VIP the iptables rule kicks in.  A backend is
+chosen (either based on session affinity or randomly) and packets are
+redirected to the backend.  Unlike the userspace proxy, packets are never
+copied to userspace, the kube-proxy does not have to be running for the VIP to
+work, and the client IP is not altered.
+
+This same basic flow executes when traffic comes in through a node-port or
+through a load-balancer, though in those cases the client IP does get altered.
 
 ## API Object
 

pkg/proxy/iptables/proxier.go

@@ -62,19 +62,24 @@ const iptablesNodePortsChain utiliptables.Chain = "KUBE-NODEPORTS"
 // the mark we apply to traffic needing SNAT
 const iptablesMasqueradeMark = "0x4d415351"
 
-// ShouldUseIptablesProxier returns true if we should use the iptables Proxier
+// IptablesVersioner can query the current iptables version.
+type IptablesVersioner interface {
+	// returns "X.Y.Z"
+	GetVersion() (string, error)
+}
+
+// CanUseIptablesProxier returns true if we should use the iptables Proxier
 // instead of the "classic" userspace Proxier.  This is determined by checking
 // the iptables version and for the existence of kernel features. It may return
 // an error if it fails to get the iptables version without error, in which
 // case it will also return false.
-func ShouldUseIptablesProxier() (bool, error) {
+func CanUseIptablesProxier(iptver IptablesVersioner) (bool, error) {
-	exec := utilexec.New()
 	minVersion, err := semver.NewVersion(iptablesMinVersion)
 	if err != nil {
 		return false, err
 	}
-	// returns "X.X.X", err
+	// returns "X.Y.Z"
-	versionString, err := utiliptables.GetIptablesVersionString(exec)
+	versionString, err := iptver.GetVersion()
 	if err != nil {
 		return false, err
 	}
@@ -89,6 +94,7 @@ func ShouldUseIptablesProxier() (bool, error) {
 	// Check for the required sysctls.  We don't care about the value, just
 	// that it exists.  If this Proxier is chosen, we'll iniialize it as we
 	// need.
+	// TODO: we should inject a sysctl.Interface like we do for iptables
 	_, err = utilsysctl.GetSysctl(sysctlRouteLocalnet)
 	if err != nil {
 		return false, err

pkg/proxy/userspace/proxier_test.go

@@ -32,7 +32,7 @@ import (
 	"k8s.io/kubernetes/pkg/proxy"
 	"k8s.io/kubernetes/pkg/types"
 	"k8s.io/kubernetes/pkg/util"
-	"k8s.io/kubernetes/pkg/util/iptables"
+	ipttest "k8s.io/kubernetes/pkg/util/iptables/testing"
 )
 
 const (
@@ -82,55 +82,6 @@ func waitForClosedPortUDP(p *Proxier, proxyPort int) error {
 	return fmt.Errorf("port %d still open", proxyPort)
 }
 
-// The iptables logic has to be tested in a proper end-to-end test, so this just stubs everything out.
-type fakeIptables struct{}
-
-func (fake *fakeIptables) EnsureChain(table iptables.Table, chain iptables.Chain) (bool, error) {
-	return false, nil
-}
-
-func (fake *fakeIptables) DeleteChain(table iptables.Table, chain iptables.Chain) error {
-	return nil
-}
-
-func (fake *fakeIptables) FlushChain(table iptables.Table, chain iptables.Chain) error {
-	return nil
-}
-
-func (fake *fakeIptables) EnsureRule(position iptables.RulePosition, table iptables.Table, chain iptables.Chain, args ...string) (bool, error) {
-	return false, nil
-}
-
-func (fake *fakeIptables) DeleteRule(table iptables.Table, chain iptables.Chain, args ...string) error {
-	return nil
-}
-
-func (fake *fakeIptables) IsIpv6() bool {
-	return false
-}
-
-func (fake *fakeIptables) Save(table iptables.Table) ([]byte, error) {
-	return []byte{}, nil
-}
-
-func (fake *fakeIptables) SaveAll() ([]byte, error) {
-	return []byte{}, nil
-}
-
-func (fake *fakeIptables) Restore(table iptables.Table, data []byte, flush iptables.FlushFlag, counters iptables.RestoreCountersFlag) error {
-	return nil
-}
-
-func (fake *fakeIptables) RestoreAll(data []byte, flush iptables.FlushFlag, counters iptables.RestoreCountersFlag) error {
-	return nil
-}
-
-func (fake *fakeIptables) AddReloadFunc(reloadFunc func()) {
-}
-
-func (fake *fakeIptables) Destroy() {
-}
-
 var tcpServerPort int
 var udpServerPort int
 
@@ -249,7 +200,7 @@ func TestTCPProxy(t *testing.T) {
 		},
 	})
 
-	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), &fakeIptables{}, net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
+	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), ipttest.NewFake(), net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -276,7 +227,7 @@ func TestUDPProxy(t *testing.T) {
 		},
 	})
 
-	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), &fakeIptables{}, net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
+	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), ipttest.NewFake(), net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -303,7 +254,7 @@ func TestUDPProxyTimeout(t *testing.T) {
 		},
 	})
 
-	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), &fakeIptables{}, net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
+	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), ipttest.NewFake(), net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -339,7 +290,7 @@ func TestMultiPortProxy(t *testing.T) {
 		}},
 	}})
 
-	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), &fakeIptables{}, net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
+	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), ipttest.NewFake(), net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -366,7 +317,7 @@ func TestMultiPortOnServiceUpdate(t *testing.T) {
 	serviceQ := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: "testnamespace", Name: "echo"}, Port: "q"}
 	serviceX := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: "testnamespace", Name: "echo"}, Port: "x"}
 
-	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), &fakeIptables{}, net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
+	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), ipttest.NewFake(), net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -429,7 +380,7 @@ func TestTCPProxyStop(t *testing.T) {
 		},
 	})
 
-	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), &fakeIptables{}, net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
+	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), ipttest.NewFake(), net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -473,7 +424,7 @@ func TestUDPProxyStop(t *testing.T) {
 		},
 	})
 
-	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), &fakeIptables{}, net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
+	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), ipttest.NewFake(), net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -511,7 +462,7 @@ func TestTCPProxyUpdateDelete(t *testing.T) {
 		},
 	})
 
-	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), &fakeIptables{}, net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
+	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), ipttest.NewFake(), net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -548,7 +499,7 @@ func TestUDPProxyUpdateDelete(t *testing.T) {
 		},
 	})
 
-	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), &fakeIptables{}, net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
+	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), ipttest.NewFake(), net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -585,7 +536,7 @@ func TestTCPProxyUpdateDeleteUpdate(t *testing.T) {
 		},
 	})
 
-	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), &fakeIptables{}, net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
+	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), ipttest.NewFake(), net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -637,7 +588,7 @@ func TestUDPProxyUpdateDeleteUpdate(t *testing.T) {
 		},
 	})
 
-	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), &fakeIptables{}, net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
+	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), ipttest.NewFake(), net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -689,7 +640,7 @@ func TestTCPProxyUpdatePort(t *testing.T) {
 		},
 	})
 
-	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), &fakeIptables{}, net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
+	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), ipttest.NewFake(), net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -737,7 +688,7 @@ func TestUDPProxyUpdatePort(t *testing.T) {
 		},
 	})
 
-	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), &fakeIptables{}, net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
+	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), ipttest.NewFake(), net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -782,7 +733,7 @@ func TestProxyUpdatePublicIPs(t *testing.T) {
 		},
 	})
 
-	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), &fakeIptables{}, net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
+	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), ipttest.NewFake(), net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -834,7 +785,7 @@ func TestProxyUpdatePortal(t *testing.T) {
 		},
 	})
 
-	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), &fakeIptables{}, net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
+	p, err := createProxier(lb, net.ParseIP("0.0.0.0"), ipttest.NewFake(), net.ParseIP("127.0.0.1"), nil, time.Minute, udpIdleTimeoutForTest)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/util/iptables/iptables.go

@@ -41,6 +41,8 @@ const (
 
 // An injectable interface for running iptables commands.  Implementations must be goroutine-safe.
 type Interface interface {
+	// GetVersion returns the "X.Y.Z" semver string for iptables.
+	GetVersion() (string, error)
 	// EnsureChain checks if the specified chain exists and, if not, creates it.  If the chain existed, return true.
 	EnsureChain(table Table, chain Chain) (bool, error)
 	// FlushChain clears the specified chain.  If the chain did not exist, return error.
@@ -135,7 +137,7 @@ type runner struct {
 
 // New returns a new Interface which will exec iptables.
 func New(exec utilexec.Interface, dbus utildbus.Interface, protocol Protocol) Interface {
-	vstring, err := GetIptablesVersionString(exec)
+	vstring, err := getIptablesVersionString(exec)
 	if err != nil {
 		glog.Warningf("Error checking iptables version, assuming version at least %s: %v", MinCheckVersion, err)
 		vstring = MinCheckVersion
@@ -186,6 +188,11 @@ func (runner *runner) connectToFirewallD() {
 	go runner.dbusSignalHandler(bus)
 }
 
+// GetVersion returns the version string.
+func (runner *runner) GetVersion() (string, error) {
+	return getIptablesVersionString(runner.exec)
+}
+
 // EnsureChain is part of Interface.
 func (runner *runner) EnsureChain(table Table, chain Chain) (bool, error) {
 	fullArgs := makeFullArgs(table, chain)
@@ -505,9 +512,9 @@ func getIptablesWaitFlag(vstring string) []string {
 	}
 }
 
-// GetIptablesVersionString runs "iptables --version" to get the version string
+// getIptablesVersionString runs "iptables --version" to get the version string
 // in the form "X.X.X"
-func GetIptablesVersionString(exec utilexec.Interface) (string, error) {
+func getIptablesVersionString(exec utilexec.Interface) (string, error) {
 	// this doesn't access mutable state so we don't need to use the interface / runner
 	bytes, err := exec.Command(cmdIptables, "--version").CombinedOutput()
 	if err != nil {

pkg/util/iptables/iptables_test.go

@@ -451,7 +451,7 @@ func TestGetIptablesHasCheckCommand(t *testing.T) {
 				func(cmd string, args ...string) exec.Cmd { return exec.InitFakeCmd(&fcmd, cmd, args...) },
 			},
 		}
-		version, err := GetIptablesVersionString(&fexec)
+		version, err := getIptablesVersionString(&fexec)
 		if (err != nil) != testCase.Err {
 			t.Errorf("Expected error: %v, Got error: %v", testCase.Err, err)
 		}

pkg/util/iptables/testing/fake.go

@@ -25,6 +25,10 @@ func NewFake() *fake {
 	return &fake{}
 }
 
+func (*fake) GetVersion() (string, error) {
+	return "0.0.0", nil
+}
+
 func (*fake) EnsureChain(table iptables.Table, chain iptables.Chain) (bool, error) {
 	return true, nil
 }