Hold node-ports for publicIPs for local IPs

This commit is contained in:
Tim Hockin 2015-08-17 23:05:05 -07:00
parent f5a9281a26
commit 5087ae6c93
2 changed files with 51 additions and 10 deletions

View File

@ -84,14 +84,16 @@ type Proxier struct {
// assert Proxier is a ProxyProvider
var _ proxy.ProxyProvider = &Proxier{}
// A key for the portMap
// A key for the portMap. The ip has to be a tring because slices can't be map
// keys.
type portMapKey struct {
ip string
port int
protocol api.Protocol
}
func (k *portMapKey) String() string {
return fmt.Sprintf("%s/%d", k.protocol, k.port)
return fmt.Sprintf("%s:%d/%s", k.ip, k.port, k.protocol)
}
// A value for the portMap
@ -459,6 +461,15 @@ func (proxier *Proxier) openPortal(service proxy.ServicePortName, info *serviceI
}
func (proxier *Proxier) openOnePortal(portal portal, protocol api.Protocol, proxyIP net.IP, proxyPort int, name proxy.ServicePortName) error {
if local, err := isLocalIP(portal.ip); err != nil {
return fmt.Errorf("can't determine if IP is local, assuming not: %v", err)
} else if local {
err := proxier.claimNodePort(portal.ip, portal.port, protocol, name)
if err != nil {
return err
}
}
// Handle traffic from containers.
args := proxier.iptablesContainerPortalArgs(portal.ip, portal.isExternal, false, portal.port, protocol, proxyIP, proxyPort, name)
existed, err := proxier.iptables.EnsureRule(iptables.Append, iptables.TableNAT, iptablesContainerPortalChain, args...)
@ -507,13 +518,13 @@ func (proxier *Proxier) openOnePortal(portal portal, protocol api.Protocol, prox
// Marks a port as being owned by a particular service, or returns error if already claimed.
// Idempotent: reclaiming with the same owner is not an error
func (proxier *Proxier) claimNodePort(port int, protocol api.Protocol, owner proxy.ServicePortName) error {
func (proxier *Proxier) claimNodePort(ip net.IP, port int, protocol api.Protocol, owner proxy.ServicePortName) error {
proxier.portMapMutex.Lock()
defer proxier.portMapMutex.Unlock()
// TODO: We could pre-populate some reserved ports into portMap and/or blacklist some well-known ports
key := portMapKey{port: port, protocol: protocol}
key := portMapKey{ip: ip.String(), port: port, protocol: protocol}
existing, found := proxier.portMap[key]
if !found {
// Hold the actual port open, even though we use iptables to redirect
@ -523,11 +534,12 @@ func (proxier *Proxier) claimNodePort(port int, protocol api.Protocol, owner pro
// it. Tools like 'ss' and 'netstat' do not show sockets that are
// bind()ed but not listen()ed, and at least the default debian netcat
// has no way to avoid about 10 seconds of retries.
socket, err := newProxySocket(protocol, net.ParseIP("127.0.0.1"), port)
socket, err := newProxySocket(protocol, ip, port)
if err != nil {
return fmt.Errorf("can't open node port for %s: %v", key.String(), err)
}
proxier.portMap[key] = &portMapValue{owner: owner, socket: socket}
glog.V(2).Infof("Claimed local port %s", key.String())
return nil
}
if existing.owner == owner {
@ -539,11 +551,11 @@ func (proxier *Proxier) claimNodePort(port int, protocol api.Protocol, owner pro
// Release a claim on a port. Returns an error if the owner does not match the claim.
// Tolerates release on an unclaimed port, to simplify .
func (proxier *Proxier) releaseNodePort(port int, protocol api.Protocol, owner proxy.ServicePortName) error {
func (proxier *Proxier) releaseNodePort(ip net.IP, port int, protocol api.Protocol, owner proxy.ServicePortName) error {
proxier.portMapMutex.Lock()
defer proxier.portMapMutex.Unlock()
key := portMapKey{port: port, protocol: protocol}
key := portMapKey{ip: ip.String(), port: port, protocol: protocol}
existing, found := proxier.portMap[key]
if !found {
// We tolerate this, it happens if we are cleaning up a failed allocation
@ -562,7 +574,7 @@ func (proxier *Proxier) openNodePort(nodePort int, protocol api.Protocol, proxyI
// TODO: Do we want to allow containers to access public services? Probably yes.
// TODO: We could refactor this to be the same code as portal, but with IP == nil
err := proxier.claimNodePort(nodePort, protocol, name)
err := proxier.claimNodePort(nil, nodePort, protocol, name)
if err != nil {
return err
}
@ -616,6 +628,14 @@ func (proxier *Proxier) closePortal(service proxy.ServicePortName, info *service
func (proxier *Proxier) closeOnePortal(portal portal, protocol api.Protocol, proxyIP net.IP, proxyPort int, name proxy.ServicePortName) []error {
el := []error{}
if local, err := isLocalIP(portal.ip); err != nil {
el = append(el, fmt.Errorf("can't determine if IP is local, assuming not: %v", err))
} else if local {
if err := proxier.releaseNodePort(nil, portal.port, protocol, name); err != nil {
el = append(el, err)
}
}
// Handle traffic from containers.
args := proxier.iptablesContainerPortalArgs(portal.ip, portal.isExternal, false, portal.port, protocol, proxyIP, proxyPort, name)
if err := proxier.iptables.DeleteRule(iptables.TableNAT, iptablesContainerPortalChain, args...); err != nil {
@ -665,13 +685,30 @@ func (proxier *Proxier) closeNodePort(nodePort int, protocol api.Protocol, proxy
el = append(el, err)
}
if err := proxier.releaseNodePort(nodePort, protocol, name); err != nil {
if err := proxier.releaseNodePort(nil, nodePort, protocol, name); err != nil {
el = append(el, err)
}
return el
}
func isLocalIP(ip net.IP) (bool, error) {
addrs, err := net.InterfaceAddrs()
if err != nil {
return false, err
}
for i := range addrs {
intf, _, err := net.ParseCIDR(addrs[i].String())
if err != nil {
return false, err
}
if ip.Equal(intf) {
return true, nil
}
}
return false, nil
}
// See comments in the *PortalArgs() functions for some details about why we
// use two chains for portals.
var iptablesContainerPortalChain iptables.Chain = "KUBE-PORTALS-CONTAINER"

View File

@ -46,7 +46,11 @@ type proxySocket interface {
}
func newProxySocket(protocol api.Protocol, ip net.IP, port int) (proxySocket, error) {
host := ip.String()
host := ""
if ip != nil {
host = ip.String()
}
switch strings.ToUpper(string(protocol)) {
case "TCP":
listener, err := net.Listen("tcp", net.JoinHostPort(host, strconv.Itoa(port)))