From 2aa52d043b022686ba4f3a1c2864c8e6f1bddf58 Mon Sep 17 00:00:00 2001 From: Brendan Burns Date: Tue, 11 Nov 2014 20:08:33 -0800 Subject: [PATCH 1/2] Add external services v2 support. --- docs/services.md | 12 ++------ pkg/api/types.go | 2 ++ pkg/api/v1beta1/conversion.go | 2 ++ pkg/api/v1beta1/types.go | 2 ++ pkg/api/v1beta2/conversion.go | 2 ++ pkg/api/v1beta2/types.go | 2 ++ pkg/api/v1beta3/types.go | 2 ++ pkg/cloudprovider/cloud.go | 4 +-- pkg/cloudprovider/fake/fake.go | 5 +-- pkg/cloudprovider/gce/gce.go | 22 ++++++++++--- pkg/proxy/proxier.go | 51 ++++++++++++++++++++++++++++++- pkg/registry/service/rest.go | 16 +++++++--- pkg/registry/service/rest_test.go | 2 +- pkg/util/iptables/iptables.go | 5 +-- 14 files changed, 104 insertions(+), 25 deletions(-) diff --git a/docs/services.md b/docs/services.md index ec58dfddeb3..788ef242c86 100644 --- a/docs/services.md +++ b/docs/services.md @@ -127,16 +127,10 @@ being aware of which `pods` they are accessing. ![Services detailed diagram](services_detail.png) +## External Services +For some parts of your application (e.g. your frontend) you want to expose a service on an external (publically visible) IP address. To achieve this, you can set the ```createExternalLoadBalancer``` flag on the service. This sets up a cloud provider specific load balancer (assuming that it is supported by your cloud provider) and also sets up IPTables rules on each host that map packets from the specified External IP address to the service proxy in the same manner as internal service IP addresses. + ## Shortcomings - -Part of the `service` specification is a `createExternalLoadBalancer` flag, -which tells the master to make an external load balancer that points to the -service. In order to do this today, the service proxy must answer on a known -(i.e. not random) port. In this case, the service port is promoted to the -proxy port. This means that it is still possible for users to collide with -each other's services or with other pods. We expect most `services` will not -set this flag, mitigating the exposure. - We expect that using iptables for portals will work at small scale, but will not scale to large clusters with thousands of services. See [the original design proposal for diff --git a/pkg/api/types.go b/pkg/api/types.go index 415ab338a85..19205e80b28 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -576,6 +576,8 @@ type ServiceSpec struct { // CreateExternalLoadBalancer indicates whether a load balancer should be created for this service. CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"` + // PublicIPs are used by external load balancers. + PublicIPs []string `json:"publicIPs,omitempty" yaml:"publicIPs,omitempty"` // ContainerPort is the name of the port on the container to direct traffic to. // Optional, if unspecified use the first port on the container. diff --git a/pkg/api/v1beta1/conversion.go b/pkg/api/v1beta1/conversion.go index 15ca3f2eafa..3208f2957be 100644 --- a/pkg/api/v1beta1/conversion.go +++ b/pkg/api/v1beta1/conversion.go @@ -370,6 +370,7 @@ func init() { return err } out.CreateExternalLoadBalancer = in.Spec.CreateExternalLoadBalancer + out.PublicIPs = in.Spec.PublicIPs out.ContainerPort = in.Spec.ContainerPort out.PortalIP = in.Spec.PortalIP out.ProxyPort = in.Spec.ProxyPort @@ -392,6 +393,7 @@ func init() { return err } out.Spec.CreateExternalLoadBalancer = in.CreateExternalLoadBalancer + out.Spec.PublicIPs = in.PublicIPs out.Spec.ContainerPort = in.ContainerPort out.Spec.PortalIP = in.PortalIP out.Spec.ProxyPort = in.ProxyPort diff --git a/pkg/api/v1beta1/types.go b/pkg/api/v1beta1/types.go index 4ff13cb9414..1e24eeabb34 100644 --- a/pkg/api/v1beta1/types.go +++ b/pkg/api/v1beta1/types.go @@ -467,6 +467,8 @@ type Service struct { // This service will route traffic to pods having labels matching this selector. Selector map[string]string `json:"selector,omitempty" yaml:"selector,omitempty"` CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"` + // PublicIPs are used by external load balancers. + PublicIPs []string `json:"publicIPs,omitempty" yaml:"publicIPs,omitempty"` // ContainerPort is the name of the port on the container to direct traffic to. // Optional, if unspecified use the first port on the container. diff --git a/pkg/api/v1beta2/conversion.go b/pkg/api/v1beta2/conversion.go index 3a34b11ea63..9ae6e30e079 100644 --- a/pkg/api/v1beta2/conversion.go +++ b/pkg/api/v1beta2/conversion.go @@ -299,6 +299,7 @@ func init() { return err } out.CreateExternalLoadBalancer = in.Spec.CreateExternalLoadBalancer + out.PublicIPs = in.Spec.PublicIPs out.ContainerPort = in.Spec.ContainerPort out.PortalIP = in.Spec.PortalIP out.ProxyPort = in.Spec.ProxyPort @@ -322,6 +323,7 @@ func init() { return err } out.Spec.CreateExternalLoadBalancer = in.CreateExternalLoadBalancer + out.Spec.PublicIPs = in.PublicIPs out.Spec.ContainerPort = in.ContainerPort out.Spec.PortalIP = in.PortalIP out.Spec.ProxyPort = in.ProxyPort diff --git a/pkg/api/v1beta2/types.go b/pkg/api/v1beta2/types.go index 378e6470d4f..21d2b196326 100644 --- a/pkg/api/v1beta2/types.go +++ b/pkg/api/v1beta2/types.go @@ -432,6 +432,8 @@ type Service struct { // This service will route traffic to pods having labels matching this selector. Selector map[string]string `json:"selector,omitempty" yaml:"selector,omitempty"` CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"` + // PublicIPs are used by external load balancers. + PublicIPs []string `json:"publicIPs,omitempty" yaml:"publicIPs,omitempty"` // ContainerPort is the name of the port on the container to direct traffic to. // Optional, if unspecified use the first port on the container. diff --git a/pkg/api/v1beta3/types.go b/pkg/api/v1beta3/types.go index 1f4fc797a2f..866ce9c97d4 100644 --- a/pkg/api/v1beta3/types.go +++ b/pkg/api/v1beta3/types.go @@ -614,6 +614,8 @@ type ServiceSpec struct { // CreateExternalLoadBalancer indicates whether a load balancer should be created for this service. CreateExternalLoadBalancer bool `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"` + // PublicIPs are used by external load balancers. + PublicIPs []string `json:"publicIPs,omitempty" yaml:"publicIPs,omitempty"` // ContainerPort is the name of the port on the container to direct traffic to. // Optional, if unspecified use the first port on the container. diff --git a/pkg/cloudprovider/cloud.go b/pkg/cloudprovider/cloud.go index 5ad76caf4d8..1db9e84670f 100644 --- a/pkg/cloudprovider/cloud.go +++ b/pkg/cloudprovider/cloud.go @@ -47,8 +47,8 @@ type TCPLoadBalancer interface { // TCPLoadBalancerExists returns whether the specified load balancer exists. // TODO: Break this up into different interfaces (LB, etc) when we have more than one type of service TCPLoadBalancerExists(name, region string) (bool, error) - // CreateTCPLoadBalancer creates a new tcp load balancer. - CreateTCPLoadBalancer(name, region string, port int, hosts []string) error + // CreateTCPLoadBalancer creates a new tcp load balancer. Returns the IP address of the balancer + CreateTCPLoadBalancer(name, region string, externalIP net.IP, port int, hosts []string) (net.IP, error) // UpdateTCPLoadBalancer updates hosts under the specified load balancer. UpdateTCPLoadBalancer(name, region string, hosts []string) error // DeleteTCPLoadBalancer deletes a specified load balancer. diff --git a/pkg/cloudprovider/fake/fake.go b/pkg/cloudprovider/fake/fake.go index 242312bbc99..1233c807557 100644 --- a/pkg/cloudprovider/fake/fake.go +++ b/pkg/cloudprovider/fake/fake.go @@ -34,6 +34,7 @@ type FakeCloud struct { NodeResources *api.NodeResources ClusterList []string MasterName string + ExternalIP net.IP cloudprovider.Zone } @@ -83,9 +84,9 @@ func (f *FakeCloud) TCPLoadBalancerExists(name, region string) (bool, error) { // CreateTCPLoadBalancer is a test-spy implementation of TCPLoadBalancer.CreateTCPLoadBalancer. // It adds an entry "create" into the internal method call record. -func (f *FakeCloud) CreateTCPLoadBalancer(name, region string, port int, hosts []string) error { +func (f *FakeCloud) CreateTCPLoadBalancer(name, region string, externalIP net.IP, port int, hosts []string) (net.IP, error) { f.addCall("create") - return f.Err + return f.ExternalIP, f.Err } // UpdateTCPLoadBalancer is a test-spy implementation of TCPLoadBalancer.UpdateTCPLoadBalancer. diff --git a/pkg/cloudprovider/gce/gce.go b/pkg/cloudprovider/gce/gce.go index 5c0c44fcac7..bcff8148320 100644 --- a/pkg/cloudprovider/gce/gce.go +++ b/pkg/cloudprovider/gce/gce.go @@ -192,10 +192,10 @@ func (gce *GCECloud) TCPLoadBalancerExists(name, region string) (bool, error) { } // CreateTCPLoadBalancer is an implementation of TCPLoadBalancer.CreateTCPLoadBalancer. -func (gce *GCECloud) CreateTCPLoadBalancer(name, region string, port int, hosts []string) error { +func (gce *GCECloud) CreateTCPLoadBalancer(name, region string, externalIP net.IP, port int, hosts []string) (net.IP, error) { pool, err := gce.makeTargetPool(name, region, hosts) if err != nil { - return err + return nil, err } req := &compute.ForwardingRule{ Name: name, @@ -203,8 +203,22 @@ func (gce *GCECloud) CreateTCPLoadBalancer(name, region string, port int, hosts PortRange: strconv.Itoa(port), Target: pool, } - _, err = gce.service.ForwardingRules.Insert(gce.projectID, region, req).Do() - return err + if len(externalIP) > 0 { + req.IPAddress = externalIP.String() + } + op, err := gce.service.ForwardingRules.Insert(gce.projectID, region, req).Do() + if err != nil { + return nil, err + } + err = gce.waitForRegionOp(op, region) + if err != nil { + return nil, err + } + fwd, err := gce.service.ForwardingRules.Get(gce.projectID, region, name).Do() + if err != nil { + return nil, err + } + return net.ParseIP(fwd.IPAddress), nil } // UpdateTCPLoadBalancer is an implementation of TCPLoadBalancer.UpdateTCPLoadBalancer. diff --git a/pkg/proxy/proxier.go b/pkg/proxy/proxier.go index 9ee86159246..45976481fda 100644 --- a/pkg/proxy/proxier.go +++ b/pkg/proxy/proxier.go @@ -40,6 +40,7 @@ type serviceInfo struct { timeout time.Duration mu sync.Mutex // protects active active bool + publicIP []string } func (si *serviceInfo) isActive() bool { @@ -443,7 +444,7 @@ func (proxier *Proxier) OnUpdate(services []api.Service) { if exists && info.isActive() && info.portalPort == service.Spec.Port && info.portalIP.Equal(serviceIP) { continue } - if exists && (info.portalPort != service.Spec.Port || !info.portalIP.Equal(serviceIP)) { + if exists && (info.portalPort != service.Spec.Port || !info.portalIP.Equal(serviceIP) || service.Spec.CreateExternalLoadBalancer != (len(info.publicIP) > 0)) { glog.V(4).Infof("Something changed for service %q: stopping it", service.Name) err := proxier.closePortal(service.Name, info) if err != nil { @@ -462,6 +463,9 @@ func (proxier *Proxier) OnUpdate(services []api.Service) { } info.portalIP = serviceIP info.portalPort = service.Spec.Port + if service.Spec.CreateExternalLoadBalancer { + info.publicIP = service.Spec.PublicIPs + } err = proxier.openPortal(service.Name, info) if err != nil { glog.Errorf("Failed to open portal for %q: %s", service.Name, err) @@ -494,6 +498,25 @@ func (proxier *Proxier) openPortal(service string, info *serviceInfo) error { if !existed { glog.Infof("Opened iptables portal for service %q on %s:%d", service, info.portalIP, info.portalPort) } + if len(info.publicIP) > 0 { + return proxier.openExternalPortal(service, info) + } + return nil +} + +func (proxier *Proxier) openExternalPortal(service string, info *serviceInfo) error { + for _, publicIP := range info.publicIP { + proxier.iptables.EnsureRule(iptables.TableNAT, iptables.ChainPostrouting, iptablesRoutingArgs(publicIP)...) + args := iptablesPortalArgs(net.ParseIP(publicIP), info.portalPort, info.protocol, proxier.listenAddress, info.proxyPort, service) + existed, err := proxier.iptables.EnsureRule(iptables.TableNAT, iptablesProxyChain, args...) + if err != nil { + glog.Errorf("Failed to install iptables %s rule for service %q", iptablesProxyChain, service) + return err + } + if !existed { + glog.Infof("Opened iptables external portal for service %q on %s:%d", service, publicIP, info.proxyPort) + } + } return nil } @@ -503,10 +526,26 @@ func (proxier *Proxier) closePortal(service string, info *serviceInfo) error { glog.Errorf("Failed to delete iptables %s rule for service %q", iptablesProxyChain, service) return err } + if len(info.publicIP) > 0 { + return proxier.closeExternalPortal(service, info) + } glog.Infof("Closed iptables portal for service %q", service) return nil } +func (proxier *Proxier) closeExternalPortal(service string, info *serviceInfo) error { + for _, publicIP := range info.publicIP { + proxier.iptables.DeleteRule(iptables.TableNAT, iptables.ChainPostrouting, iptablesRoutingArgs(publicIP)...) + args := iptablesPortalArgs(net.ParseIP(publicIP), info.portalPort, info.protocol, proxier.listenAddress, info.proxyPort, service) + if err := proxier.iptables.DeleteRule(iptables.TableNAT, iptablesProxyChain, args...); err != nil { + glog.Errorf("Failed to delete external iptables %s rule for service %q", iptablesProxyChain, service) + return err + } + } + glog.Infof("Closed external iptables portal for service %q", service) + return nil +} + var iptablesProxyChain iptables.Chain = "KUBE-PROXY" // Ensure that the iptables infrastructure we use is set up. This can safely be called periodically. @@ -538,6 +577,16 @@ var localhostIPv4 = net.ParseIP("127.0.0.1") var zeroIPv6 = net.ParseIP("::0") var localhostIPv6 = net.ParseIP("::1") +// Build an iptables args to route in a specific external ip +func iptablesRoutingArgs(destIP string) []string { + return []string{ + "!", + "-d", destIP + "/32", + "-o", "eth0", + "-j", "MASQUERADE", + } +} + // Build a slice of iptables args for a portal rule. func iptablesPortalArgs(destIP net.IP, destPort int, protocol api.Protocol, proxyIP net.IP, proxyPort int, service string) []string { args := []string{ diff --git a/pkg/registry/service/rest.go b/pkg/registry/service/rest.go index b85123e3ac9..1346faee37b 100644 --- a/pkg/registry/service/rest.go +++ b/pkg/registry/service/rest.go @@ -133,13 +133,21 @@ func (rs *REST) Create(ctx api.Context, obj runtime.Object) (<-chan apiserver.RE if err != nil { return nil, err } - err = balancer.CreateTCPLoadBalancer(service.Name, zone.Region, service.Spec.Port, hostsFromMinionList(hosts)) + var ip net.IP + if len(service.Spec.PublicIPs) > 0 { + for _, publicIP := range service.Spec.PublicIPs { + ip, err = balancer.CreateTCPLoadBalancer(service.Name, zone.Region, net.ParseIP(publicIP), service.Spec.Port, hostsFromMinionList(hosts)) + if err != nil { + break + } + } + } else { + ip, err = balancer.CreateTCPLoadBalancer(service.Name, zone.Region, nil, service.Spec.Port, hostsFromMinionList(hosts)) + } if err != nil { return nil, err } - // External load-balancers require a known port for the service proxy. - // TODO: If we end up brokering HostPorts between Pods and Services, this can be any port. - service.Spec.ProxyPort = service.Spec.Port + service.Spec.PublicIPs = []string{ip.String()} } err := rs.registry.CreateService(ctx, service) if err != nil { diff --git a/pkg/registry/service/rest_test.go b/pkg/registry/service/rest_test.go index b970c58b26b..7f6451196a0 100644 --- a/pkg/registry/service/rest_test.go +++ b/pkg/registry/service/rest_test.go @@ -638,7 +638,7 @@ func TestServiceRegistryIPExternalLoadBalancer(t *testing.T) { if created_service.Spec.PortalIP != "1.2.3.1" { t.Errorf("Unexpected PortalIP: %s", created_service.Spec.PortalIP) } - if created_service.Spec.ProxyPort != 6502 { + if created_service.Spec.ProxyPort != 0 { t.Errorf("Unexpected ProxyPort: %d", created_service.Spec.ProxyPort) } } diff --git a/pkg/util/iptables/iptables.go b/pkg/util/iptables/iptables.go index d4dab06df32..e410d3dd9b5 100644 --- a/pkg/util/iptables/iptables.go +++ b/pkg/util/iptables/iptables.go @@ -54,8 +54,9 @@ const ( type Chain string const ( - ChainPrerouting Chain = "PREROUTING" - ChainOutput Chain = "OUTPUT" + ChainPostrouting Chain = "POSTROUTING" + ChainPrerouting Chain = "PREROUTING" + ChainOutput Chain = "OUTPUT" ) // runner implements Interface in terms of exec("iptables"). From 4a8a2b5a9f87413a3284caa6de7f99ca4edf4921 Mon Sep 17 00:00:00 2001 From: Brendan Burns Date: Thu, 13 Nov 2014 22:23:33 -0800 Subject: [PATCH 2/2] Address comments. --- pkg/proxy/proxier.go | 15 ++------------- pkg/registry/service/rest.go | 1 + 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/pkg/proxy/proxier.go b/pkg/proxy/proxier.go index 45976481fda..0e2ae60367f 100644 --- a/pkg/proxy/proxier.go +++ b/pkg/proxy/proxier.go @@ -40,7 +40,8 @@ type serviceInfo struct { timeout time.Duration mu sync.Mutex // protects active active bool - publicIP []string + // TODO: make this an net.IP address + publicIP []string } func (si *serviceInfo) isActive() bool { @@ -506,7 +507,6 @@ func (proxier *Proxier) openPortal(service string, info *serviceInfo) error { func (proxier *Proxier) openExternalPortal(service string, info *serviceInfo) error { for _, publicIP := range info.publicIP { - proxier.iptables.EnsureRule(iptables.TableNAT, iptables.ChainPostrouting, iptablesRoutingArgs(publicIP)...) args := iptablesPortalArgs(net.ParseIP(publicIP), info.portalPort, info.protocol, proxier.listenAddress, info.proxyPort, service) existed, err := proxier.iptables.EnsureRule(iptables.TableNAT, iptablesProxyChain, args...) if err != nil { @@ -535,7 +535,6 @@ func (proxier *Proxier) closePortal(service string, info *serviceInfo) error { func (proxier *Proxier) closeExternalPortal(service string, info *serviceInfo) error { for _, publicIP := range info.publicIP { - proxier.iptables.DeleteRule(iptables.TableNAT, iptables.ChainPostrouting, iptablesRoutingArgs(publicIP)...) args := iptablesPortalArgs(net.ParseIP(publicIP), info.portalPort, info.protocol, proxier.listenAddress, info.proxyPort, service) if err := proxier.iptables.DeleteRule(iptables.TableNAT, iptablesProxyChain, args...); err != nil { glog.Errorf("Failed to delete external iptables %s rule for service %q", iptablesProxyChain, service) @@ -577,16 +576,6 @@ var localhostIPv4 = net.ParseIP("127.0.0.1") var zeroIPv6 = net.ParseIP("::0") var localhostIPv6 = net.ParseIP("::1") -// Build an iptables args to route in a specific external ip -func iptablesRoutingArgs(destIP string) []string { - return []string{ - "!", - "-d", destIP + "/32", - "-o", "eth0", - "-j", "MASQUERADE", - } -} - // Build a slice of iptables args for a portal rule. func iptablesPortalArgs(destIP net.IP, destPort int, protocol api.Protocol, proxyIP net.IP, proxyPort int, service string) []string { args := []string{ diff --git a/pkg/registry/service/rest.go b/pkg/registry/service/rest.go index 1346faee37b..af6175ffb74 100644 --- a/pkg/registry/service/rest.go +++ b/pkg/registry/service/rest.go @@ -112,6 +112,7 @@ func (rs *REST) Create(ctx api.Context, obj runtime.Object) (<-chan apiserver.RE return apiserver.MakeAsync(func() (runtime.Object, error) { // TODO: Consider moving this to a rectification loop, so that we make/remove external load balancers // correctly no matter what http operations happen. + // TODO: Get rid of ProxyPort. service.Spec.ProxyPort = 0 if service.Spec.CreateExternalLoadBalancer { if rs.cloud == nil {