github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-24 12:15:52 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #98755 from aojea/hostportdockershim

Browse Source 
dockershim hostport manager use HostIP
This commit is contained in:
Kubernetes Prow Robot 2021-02-05 14:06:42 -08:00 committed by GitHub
commit 0faf096041
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 271 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
pkg/kubelet/dockershim/network/hostport/fake_iptables.go
View File

@ -148,14 +148,14 @@ func (f *fakeIPTables) ensureRule(position utiliptables.RulePosition, tableName
return true, nil
}
if position == utiliptables.Prepend {
switch position {
case utiliptables.Prepend:
chain.rules = append([]string{rule}, chain.rules...)
} else if position == utiliptables.Append {
case utiliptables.Append:
chain.rules = append(chain.rules, rule)
} else {
default:
return false, fmt.Errorf("unknown position argument %q", position)
}
return false, nil
}
@ -185,7 +185,7 @@ func normalizeRule(rule string) (string, error) {
// Normalize un-prefixed IP addresses like iptables does
if net.ParseIP(arg) != nil {
arg = arg + "/32"
arg += "/32"
}
if len(normalized) > 0 {
@ -281,7 +281,10 @@ func (f *fakeIPTables) restore(restoreTableName utiliptables.Table, data []byte,
if strings.HasPrefix(line, ":") {
chainName := utiliptables.Chain(strings.Split(line[1:], " ")[0])
if flush == utiliptables.FlushTables {
table, chain, _ := f.getChain(tableName, chainName)
table, chain, err := f.getChain(tableName, chainName)
if err != nil {
return err
}
if chain != nil {
delete(table.chains, string(chainName))
}

32
pkg/kubelet/dockershim/network/hostport/hostport.go
View File

@ -21,6 +21,7 @@ package hostport
import (
"fmt"
"net"
"strconv"
"strings"
"k8s.io/klog/v2"
@ -53,7 +54,18 @@ type PodPortMapping struct {
IP net.IP
}
// ipFamily refers to a specific family if not empty, i.e. "4" or "6".
type ipFamily string
// Constants for valid IPFamily:
const (
IPv4 ipFamily = "4"
IPv6 ipFamily = "6"
)
type hostport struct {
ipFamily ipFamily
ip string
port int32
protocol string
}
@ -78,19 +90,23 @@ func openLocalPort(hp *hostport) (closeable, error) {
// bind()ed but not listen()ed, and at least the default debian netcat
// has no way to avoid about 10 seconds of retries.
var socket closeable
// open the socket on the HostIP and HostPort specified
address := net.JoinHostPort(hp.ip, strconv.Itoa(int(hp.port)))
switch hp.protocol {
case "tcp":
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", hp.port))
network := "tcp" + string(hp.ipFamily)
listener, err := net.Listen(network, address)
if err != nil {
return nil, err
}
socket = listener
case "udp":
addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf(":%d", hp.port))
network := "udp" + string(hp.ipFamily)
addr, err := net.ResolveUDPAddr(network, address)
if err != nil {
return nil, err
}
conn, err := net.ListenUDP("udp", addr)
conn, err := net.ListenUDP(network, addr)
if err != nil {
return nil, err
}
@ -103,8 +119,10 @@ func openLocalPort(hp *hostport) (closeable, error) {
}
// portMappingToHostport creates hostport structure based on input portmapping
func portMappingToHostport(portMapping *PortMapping) hostport {
func portMappingToHostport(portMapping *PortMapping, family ipFamily) hostport {
return hostport{
ipFamily: family,
ip: portMapping.HostIP,
port: portMapping.HostPort,
protocol: strings.ToLower(string(portMapping.Protocol)),
}
@ -124,9 +142,11 @@ func ensureKubeHostportChains(iptables utiliptables.Interface, natInterfaceName
{utiliptables.TableNAT, utiliptables.ChainOutput},
{utiliptables.TableNAT, utiliptables.ChainPrerouting},
}
args := []string{"-m", "comment", "--comment", "kube hostport portals",
args := []string{
"-m", "comment", "--comment", "kube hostport portals",
"-m", "addrtype", "--dst-type", "LOCAL",
"-j", string(kubeHostportsChain)}
"-j", string(kubeHostportsChain),
}
for _, tc := range tableChainsNeedJumpServices {
// KUBE-HOSTPORTS chain needs to be appended to the system chains.
// This ensures KUBE-SERVICES chain gets processed first.

71
pkg/kubelet/dockershim/network/hostport/hostport_manager.go
View File

@ -59,6 +59,7 @@ type hostportManager struct {
mu sync.Mutex
}
// NewHostportManager creates a new HostPortManager
func NewHostportManager(iptables utiliptables.Interface) HostPortManager {
h := &hostportManager{
hostPortMap: make(map[hostport]closeable),
@ -78,13 +79,6 @@ func (hm *hostportManager) Add(id string, podPortMapping *PodPortMapping, natInt
return nil
}
podFullName := getPodFullName(podPortMapping)
// skip if there is no hostport needed
hostportMappings := gatherHostportMappings(podPortMapping)
if len(hostportMappings) == 0 {
return nil
}
// IP.To16() returns nil if IP is not a valid IPv4 or IPv6 address
if podPortMapping.IP.To16() == nil {
return fmt.Errorf("invalid or missing IP of pod %s", podFullName)
@ -92,11 +86,17 @@ func (hm *hostportManager) Add(id string, podPortMapping *PodPortMapping, natInt
podIP := podPortMapping.IP.String()
isIPv6 := utilnet.IsIPv6(podPortMapping.IP)
// skip if there is no hostport needed
hostportMappings := gatherHostportMappings(podPortMapping, isIPv6)
if len(hostportMappings) == 0 {
return nil
}
if isIPv6 != hm.iptables.IsIPv6() {
return fmt.Errorf("HostPortManager IP family mismatch: %v, isIPv6 - %v", podIP, isIPv6)
}
if err = ensureKubeHostportChains(hm.iptables, natInterfaceName); err != nil {
if err := ensureKubeHostportChains(hm.iptables, natInterfaceName); err != nil {
return err
}
@ -152,10 +152,17 @@ func (hm *hostportManager) Add(id string, podPortMapping *PodPortMapping, natInt
// DNAT to the podIP:containerPort
hostPortBinding := net.JoinHostPort(podIP, strconv.Itoa(int(pm.ContainerPort)))
writeLine(natRules, "-A", string(chain),
"-m", "comment", "--comment", fmt.Sprintf(`"%s hostport %d"`, podFullName, pm.HostPort),
"-m", protocol, "-p", protocol,
"-j", "DNAT", fmt.Sprintf("--to-destination=%s", hostPortBinding))
if pm.HostIP == "" || pm.HostIP == "0.0.0.0" || pm.HostIP == "::" {
writeLine(natRules, "-A", string(chain),
"-m", "comment", "--comment", fmt.Sprintf(`"%s hostport %d"`, podFullName, pm.HostPort),
"-m", protocol, "-p", protocol,
"-j", "DNAT", fmt.Sprintf("--to-destination=%s", hostPortBinding))
} else {
writeLine(natRules, "-A", string(chain),
"-m", "comment", "--comment", fmt.Sprintf(`"%s hostport %d"`, podFullName, pm.HostPort),
"-m", protocol, "-p", protocol, "-d", pm.HostIP,
"-j", "DNAT", fmt.Sprintf("--to-destination=%s", hostPortBinding))
}
}
// getHostportChain should be able to provide unique hostport chain name using hash
@ -198,8 +205,8 @@ func (hm *hostportManager) Remove(id string, podPortMapping *PodPortMapping) (er
return nil
}
hostportMappings := gatherHostportMappings(podPortMapping)
if len(hostportMappings) <= 0 {
hostportMappings := gatherHostportMappings(podPortMapping, hm.iptables.IsIPv6())
if len(hostportMappings) == 0 {
return nil
}
@ -231,6 +238,12 @@ func (hm *hostportManager) Remove(id string, podPortMapping *PodPortMapping) (er
}
}
// exit if there is nothing to remove
// don´t forget to clean up opened pod host ports
if len(existingChainsToRemove) == 0 {
return hm.closeHostports(hostportMappings)
}
natChains := bytes.NewBuffer(nil)
natRules := bytes.NewBuffer(nil)
writeLine(natChains, "*nat")
@ -245,7 +258,7 @@ func (hm *hostportManager) Remove(id string, podPortMapping *PodPortMapping) (er
}
writeLine(natRules, "COMMIT")
if err = hm.syncIPTables(append(natChains.Bytes(), natRules.Bytes()...)); err != nil {
if err := hm.syncIPTables(append(natChains.Bytes(), natRules.Bytes()...)); err != nil {
return err
}
@ -279,7 +292,12 @@ func (hm *hostportManager) openHostports(podPortMapping *PodPortMapping) (map[ho
continue
}
hp := portMappingToHostport(pm)
// HostIP IP family is not handled by this port opener
if pm.HostIP != "" && utilnet.IsIPv6String(pm.HostIP) != hm.iptables.IsIPv6() {
continue
}
hp := portMappingToHostport(pm, hm.getIPFamily())
socket, err := hm.portOpener(&hp)
if err != nil {
retErr = fmt.Errorf("cannot open hostport %d for pod %s: %v", pm.HostPort, getPodFullName(podPortMapping), err)
@ -304,7 +322,7 @@ func (hm *hostportManager) openHostports(podPortMapping *PodPortMapping) (map[ho
func (hm *hostportManager) closeHostports(hostportMappings []*PortMapping) error {
errList := []error{}
for _, pm := range hostportMappings {
hp := portMappingToHostport(pm)
hp := portMappingToHostport(pm, hm.getIPFamily())
if socket, ok := hm.hostPortMap[hp]; ok {
klog.V(2).Infof("Closing host port %s", hp.String())
if err := socket.Close(); err != nil {
@ -312,11 +330,22 @@ func (hm *hostportManager) closeHostports(hostportMappings []*PortMapping) error
continue
}
delete(hm.hostPortMap, hp)
} else {
klog.V(5).Infof("host port %s does not have an open socket", hp.String())
}
}
return utilerrors.NewAggregate(errList)
}
// getIPFamily returns the hostPortManager IP family
func (hm *hostportManager) getIPFamily() ipFamily {
family := IPv4
if hm.iptables.IsIPv6() {
family = IPv6
}
return family
}
// getHostportChain takes id, hostport and protocol for a pod and returns associated iptables chain.
// This is computed by hashing (sha256) then encoding to base32 and truncating with the prefix
// "KUBE-HP-". We do this because IPTables Chain Names must be <= 28 chars long, and the longer
@ -324,18 +353,22 @@ func (hm *hostportManager) closeHostports(hostportMappings []*PortMapping) error
// WARNING: Please do not change this function. Otherwise, HostportManager may not be able to
// identify existing iptables chains.
func getHostportChain(id string, pm *PortMapping) utiliptables.Chain {
hash := sha256.Sum256([]byte(id + strconv.Itoa(int(pm.HostPort)) + string(pm.Protocol)))
hash := sha256.Sum256([]byte(id + strconv.Itoa(int(pm.HostPort)) + string(pm.Protocol) + pm.HostIP))
encoded := base32.StdEncoding.EncodeToString(hash[:])
return utiliptables.Chain(kubeHostportChainPrefix + encoded[:16])
}
// gatherHostportMappings returns all the PortMappings which has hostport for a pod
func gatherHostportMappings(podPortMapping *PodPortMapping) []*PortMapping {
// it filters the PortMappings that use HostIP and doesn't match the IP family specified
func gatherHostportMappings(podPortMapping *PodPortMapping, isIPv6 bool) []*PortMapping {
mappings := []*PortMapping{}
for _, pm := range podPortMapping.PortMappings {
if pm.HostPort <= 0 {
continue
}
if pm.HostIP != "" && utilnet.IsIPv6String(pm.HostIP) != isIPv6 {
continue
}
mappings = append(mappings, pm)
}
return mappings

201
pkg/kubelet/dockershim/network/hostport/hostport_manager_test.go
View File

@ -35,6 +35,7 @@ func TestOpenCloseHostports(t *testing.T) {
podPortMapping *PodPortMapping
expectError bool
}{
// no portmaps
{
&PodPortMapping{
Namespace: "ns1",
@ -42,6 +43,7 @@ func TestOpenCloseHostports(t *testing.T) {
},
false,
},
// allocate port 80/TCP, 8080/TCP and 443/TCP
{
&PodPortMapping{
Namespace: "ns1",
@ -54,6 +56,7 @@ func TestOpenCloseHostports(t *testing.T) {
},
false,
},
// fail to allocate port previously allocated 80/TCP
{
&PodPortMapping{
Namespace: "ns1",
@ -64,6 +67,7 @@ func TestOpenCloseHostports(t *testing.T) {
},
true,
},
// fail to allocate port previously allocated 8080/TCP
{
&PodPortMapping{
Namespace: "ns1",
@ -75,6 +79,7 @@ func TestOpenCloseHostports(t *testing.T) {
},
true,
},
// allocate port 8081/TCP
{
&PodPortMapping{
Namespace: "ns1",
@ -85,6 +90,7 @@ func TestOpenCloseHostports(t *testing.T) {
},
false,
},
// allocate port 7777/SCTP
{
&PodPortMapping{
Namespace: "ns1",
@ -95,9 +101,34 @@ func TestOpenCloseHostports(t *testing.T) {
},
false,
},
// same HostPort different HostIP
{
&PodPortMapping{
Namespace: "ns1",
Name: "n5",
PortMappings: []*PortMapping{
{HostPort: 8888, Protocol: v1.ProtocolUDP, HostIP: "127.0.0.1"},
{HostPort: 8888, Protocol: v1.ProtocolUDP, HostIP: "127.0.0.2"},
},
},
false,
},
// same HostPort different protocol
{
&PodPortMapping{
Namespace: "ns1",
Name: "n6",
PortMappings: []*PortMapping{
{HostPort: 9999, Protocol: v1.ProtocolTCP},
{HostPort: 9999, Protocol: v1.ProtocolUDP},
},
},
false,
},
}
iptables := NewFakeIPTables()
iptables.protocol = utiliptables.ProtocolIPv4
portOpener := NewFakeSocketManager()
manager := &hostportManager{
hostPortMap: make(map[hostport]closeable),
@ -106,23 +137,30 @@ func TestOpenCloseHostports(t *testing.T) {
execer: exec.New(),
}
// open all hostports defined in the test cases
for _, tc := range openPortCases {
mapping, err := manager.openHostports(tc.podPortMapping)
for hostport, socket := range mapping {
manager.hostPortMap[hostport] = socket
}
if tc.expectError {
assert.Error(t, err)
continue
}
assert.NoError(t, err)
// SCTP ports are not allocated
countSctp := 0
for _, pm := range tc.podPortMapping.PortMappings {
if pm.Protocol == v1.ProtocolSCTP {
countSctp += 1
countSctp++
}
}
assert.EqualValues(t, len(mapping), len(tc.podPortMapping.PortMappings)-countSctp)
}
// We have 4 ports: 80, 443, 8080, 8081 open now.
// We have following ports open: 80/TCP, 443/TCP, 8080/TCP, 8081/TCP,
// 127.0.0.1:8888/TCP, 127.0.0.2:8888/TCP, 9999/TCP and 9999/UDP open now.
assert.EqualValues(t, len(manager.hostPortMap), 8)
closePortCases := []struct {
portMappings []*PortMapping
expectError bool
@ -165,8 +203,21 @@ func TestOpenCloseHostports(t *testing.T) {
{HostPort: 7777, Protocol: v1.ProtocolSCTP},
},
},
{
portMappings: []*PortMapping{
{HostPort: 8888, Protocol: v1.ProtocolUDP, HostIP: "127.0.0.1"},
{HostPort: 8888, Protocol: v1.ProtocolUDP, HostIP: "127.0.0.2"},
},
},
{
portMappings: []*PortMapping{
{HostPort: 9999, Protocol: v1.ProtocolTCP},
{HostPort: 9999, Protocol: v1.ProtocolUDP},
},
},
}
// close all the hostports opened in previous step
for _, tc := range closePortCases {
err := manager.closeHostports(tc.portMappings)
if tc.expectError {
@ -175,12 +226,13 @@ func TestOpenCloseHostports(t *testing.T) {
}
assert.NoError(t, err)
}
// Clear all elements in hostPortMap
// assert all elements in hostPortMap were cleared
assert.Zero(t, len(manager.hostPortMap))
}
func TestHostportManager(t *testing.T) {
iptables := NewFakeIPTables()
iptables.protocol = utiliptables.ProtocolIPv4
portOpener := NewFakeSocketManager()
manager := &hostportManager{
hostPortMap: make(map[hostport]closeable),
@ -188,11 +240,11 @@ func TestHostportManager(t *testing.T) {
portOpener: portOpener.openFakeSocket,
execer: exec.New(),
}
testCases := []struct {
mapping *PodPortMapping
expectError bool
}{
// open HostPorts 8080/TCP, 8081/UDP and 8083/SCTP
{
mapping: &PodPortMapping{
Name: "pod1",
@ -219,6 +271,7 @@ func TestHostportManager(t *testing.T) {
},
expectError: false,
},
// fail to open HostPort due to conflict 8083/SCTP
{
mapping: &PodPortMapping{
Name: "pod2",
@ -245,6 +298,7 @@ func TestHostportManager(t *testing.T) {
},
expectError: true,
},
// open port 443
{
mapping: &PodPortMapping{
Name: "pod3",
@ -261,11 +315,12 @@ func TestHostportManager(t *testing.T) {
},
expectError: false,
},
// fail to open HostPort 8443 already allocated
{
mapping: &PodPortMapping{
Name: "pod3",
Namespace: "ns1",
IP: net.ParseIP("2001:beef::2"),
IP: net.ParseIP("192.168.12.12"),
HostNetwork: false,
PortMappings: []*PortMapping{
{
@ -277,6 +332,71 @@ func TestHostportManager(t *testing.T) {
},
expectError: true,
},
// skip HostPort with PodIP and HostIP using different families
{
mapping: &PodPortMapping{
Name: "pod4",
Namespace: "ns1",
IP: net.ParseIP("2001:beef::2"),
HostNetwork: false,
PortMappings: []*PortMapping{
{
HostPort: 8444,
ContainerPort: 444,
Protocol: v1.ProtocolTCP,
HostIP: "192.168.1.1",
},
},
},
expectError: false,
},
// open same HostPort on different IP
{
mapping: &PodPortMapping{
Name: "pod5",
Namespace: "ns5",
IP: net.ParseIP("10.1.1.5"),
HostNetwork: false,
PortMappings: []*PortMapping{
{
HostPort: 8888,
ContainerPort: 443,
Protocol: v1.ProtocolTCP,
HostIP: "127.0.0.2",
},
{
HostPort: 8888,
ContainerPort: 443,
Protocol: v1.ProtocolTCP,
HostIP: "127.0.0.1",
},
},
},
expectError: false,
},
// open same HostPort on different
{
mapping: &PodPortMapping{
Name: "pod6",
Namespace: "ns1",
IP: net.ParseIP("10.1.1.2"),
HostNetwork: false,
PortMappings: []*PortMapping{
{
HostPort: 9999,
ContainerPort: 443,
Protocol: v1.ProtocolTCP,
},
{
HostPort: 9999,
ContainerPort: 443,
Protocol: v1.ProtocolUDP,
},
},
},
expectError: false,
},
}
// Add Hostports
@ -290,7 +410,15 @@ func TestHostportManager(t *testing.T) {
}
// Check port opened
expectedPorts := []hostport{{8080, "tcp"}, {8081, "udp"}, {8443, "tcp"}}
expectedPorts := []hostport{
{IPv4, "", 8080, "tcp"},
{IPv4, "", 8081, "udp"},
{IPv4, "", 8443, "tcp"},
{IPv4, "127.0.0.1", 8888, "tcp"},
{IPv4, "127.0.0.2", 8888, "tcp"},
{IPv4, "", 9999, "tcp"},
{IPv4, "", 9999, "udp"},
}
openedPorts := make(map[hostport]bool)
for hp, port := range portOpener.mem {
if !port.closed {
@ -319,24 +447,41 @@ func TestHostportManager(t *testing.T) {
`:KUBE-HP-63UPIDJXVRSZGSUZ - [0:0]`: true,
`:KUBE-HP-WFBOALXEP42XEMJK - [0:0]`: true,
`:KUBE-HP-XU6AWMMJYOZOFTFZ - [0:0]`: true,
"-A KUBE-HOSTPORTS -m comment --comment \"pod3_ns1 hostport 8443\" -m tcp -p tcp --dport 8443 -j KUBE-HP-WFBOALXEP42XEMJK": true,
"-A KUBE-HOSTPORTS -m comment --comment \"pod1_ns1 hostport 8081\" -m udp -p udp --dport 8081 -j KUBE-HP-63UPIDJXVRSZGSUZ": true,
"-A KUBE-HOSTPORTS -m comment --comment \"pod1_ns1 hostport 8080\" -m tcp -p tcp --dport 8080 -j KUBE-HP-IJHALPHTORMHHPPK": true,
"-A KUBE-HOSTPORTS -m comment --comment \"pod1_ns1 hostport 8083\" -m sctp -p sctp --dport 8083 -j KUBE-HP-XU6AWMMJYOZOFTFZ": true,
"-A OUTPUT -m comment --comment \"kube hostport portals\" -m addrtype --dst-type LOCAL -j KUBE-HOSTPORTS": true,
"-A PREROUTING -m comment --comment \"kube hostport portals\" -m addrtype --dst-type LOCAL -j KUBE-HOSTPORTS": true,
"-A POSTROUTING -m comment --comment \"SNAT for localhost access to hostports\" -o cbr0 -s 127.0.0.0/8 -j MASQUERADE": true,
"-A KUBE-HP-IJHALPHTORMHHPPK -m comment --comment \"pod1_ns1 hostport 8080\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ": true,
"-A KUBE-HP-IJHALPHTORMHHPPK -m comment --comment \"pod1_ns1 hostport 8080\" -m tcp -p tcp -j DNAT --to-destination 10.1.1.2:80": true,
"-A KUBE-HP-63UPIDJXVRSZGSUZ -m comment --comment \"pod1_ns1 hostport 8081\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ": true,
"-A KUBE-HP-63UPIDJXVRSZGSUZ -m comment --comment \"pod1_ns1 hostport 8081\" -m udp -p udp -j DNAT --to-destination 10.1.1.2:81": true,
"-A KUBE-HP-XU6AWMMJYOZOFTFZ -m comment --comment \"pod1_ns1 hostport 8083\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ": true,
"-A KUBE-HP-XU6AWMMJYOZOFTFZ -m comment --comment \"pod1_ns1 hostport 8083\" -m sctp -p sctp -j DNAT --to-destination 10.1.1.2:83": true,
"-A KUBE-HP-WFBOALXEP42XEMJK -m comment --comment \"pod3_ns1 hostport 8443\" -s 10.1.1.4/32 -j KUBE-MARK-MASQ": true,
"-A KUBE-HP-WFBOALXEP42XEMJK -m comment --comment \"pod3_ns1 hostport 8443\" -m tcp -p tcp -j DNAT --to-destination 10.1.1.4:443": true,
`:KUBE-HP-TUKTZ736U5JD5UTK - [0:0]`: true,
`:KUBE-HP-CAAJ45HDITK7ARGM - [0:0]`: true,
`:KUBE-HP-WFUNFVXVDLD5ZVXN - [0:0]`: true,
`:KUBE-HP-4MFWH2F2NAOMYD6A - [0:0]`: true,
"-A KUBE-HOSTPORTS -m comment --comment \"pod3_ns1 hostport 8443\" -m tcp -p tcp --dport 8443 -j KUBE-HP-WFBOALXEP42XEMJK": true,
"-A KUBE-HOSTPORTS -m comment --comment \"pod1_ns1 hostport 8081\" -m udp -p udp --dport 8081 -j KUBE-HP-63UPIDJXVRSZGSUZ": true,
"-A KUBE-HOSTPORTS -m comment --comment \"pod1_ns1 hostport 8080\" -m tcp -p tcp --dport 8080 -j KUBE-HP-IJHALPHTORMHHPPK": true,
"-A KUBE-HOSTPORTS -m comment --comment \"pod1_ns1 hostport 8083\" -m sctp -p sctp --dport 8083 -j KUBE-HP-XU6AWMMJYOZOFTFZ": true,
"-A KUBE-HOSTPORTS -m comment --comment \"pod5_ns5 hostport 8888\" -m tcp -p tcp --dport 8888 -j KUBE-HP-TUKTZ736U5JD5UTK": true,
"-A KUBE-HOSTPORTS -m comment --comment \"pod5_ns5 hostport 8888\" -m tcp -p tcp --dport 8888 -j KUBE-HP-CAAJ45HDITK7ARGM": true,
"-A KUBE-HOSTPORTS -m comment --comment \"pod6_ns1 hostport 9999\" -m udp -p udp --dport 9999 -j KUBE-HP-4MFWH2F2NAOMYD6A": true,
"-A KUBE-HOSTPORTS -m comment --comment \"pod6_ns1 hostport 9999\" -m tcp -p tcp --dport 9999 -j KUBE-HP-WFUNFVXVDLD5ZVXN": true,
"-A OUTPUT -m comment --comment \"kube hostport portals\" -m addrtype --dst-type LOCAL -j KUBE-HOSTPORTS": true,
"-A PREROUTING -m comment --comment \"kube hostport portals\" -m addrtype --dst-type LOCAL -j KUBE-HOSTPORTS": true,
"-A POSTROUTING -m comment --comment \"SNAT for localhost access to hostports\" -o cbr0 -s 127.0.0.0/8 -j MASQUERADE": true,
"-A KUBE-HP-IJHALPHTORMHHPPK -m comment --comment \"pod1_ns1 hostport 8080\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ": true,
"-A KUBE-HP-IJHALPHTORMHHPPK -m comment --comment \"pod1_ns1 hostport 8080\" -m tcp -p tcp -j DNAT --to-destination 10.1.1.2:80": true,
"-A KUBE-HP-63UPIDJXVRSZGSUZ -m comment --comment \"pod1_ns1 hostport 8081\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ": true,
"-A KUBE-HP-63UPIDJXVRSZGSUZ -m comment --comment \"pod1_ns1 hostport 8081\" -m udp -p udp -j DNAT --to-destination 10.1.1.2:81": true,
"-A KUBE-HP-XU6AWMMJYOZOFTFZ -m comment --comment \"pod1_ns1 hostport 8083\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ": true,
"-A KUBE-HP-XU6AWMMJYOZOFTFZ -m comment --comment \"pod1_ns1 hostport 8083\" -m sctp -p sctp -j DNAT --to-destination 10.1.1.2:83": true,
"-A KUBE-HP-WFBOALXEP42XEMJK -m comment --comment \"pod3_ns1 hostport 8443\" -s 10.1.1.4/32 -j KUBE-MARK-MASQ": true,
"-A KUBE-HP-WFBOALXEP42XEMJK -m comment --comment \"pod3_ns1 hostport 8443\" -m tcp -p tcp -j DNAT --to-destination 10.1.1.4:443": true,
"-A KUBE-HP-TUKTZ736U5JD5UTK -m comment --comment \"pod5_ns5 hostport 8888\" -s 10.1.1.5/32 -j KUBE-MARK-MASQ": true,
"-A KUBE-HP-TUKTZ736U5JD5UTK -m comment --comment \"pod5_ns5 hostport 8888\" -m tcp -p tcp -d 127.0.0.1/32 -j DNAT --to-destination 10.1.1.5:443": true,
"-A KUBE-HP-CAAJ45HDITK7ARGM -m comment --comment \"pod5_ns5 hostport 8888\" -s 10.1.1.5/32 -j KUBE-MARK-MASQ": true,
"-A KUBE-HP-CAAJ45HDITK7ARGM -m comment --comment \"pod5_ns5 hostport 8888\" -m tcp -p tcp -d 127.0.0.2/32 -j DNAT --to-destination 10.1.1.5:443": true,
"-A KUBE-HP-WFUNFVXVDLD5ZVXN -m comment --comment \"pod6_ns1 hostport 9999\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ": true,
"-A KUBE-HP-WFUNFVXVDLD5ZVXN -m comment --comment \"pod6_ns1 hostport 9999\" -m tcp -p tcp -j DNAT --to-destination 10.1.1.2:443": true,
"-A KUBE-HP-4MFWH2F2NAOMYD6A -m comment --comment \"pod6_ns1 hostport 9999\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ": true,
"-A KUBE-HP-4MFWH2F2NAOMYD6A -m comment --comment \"pod6_ns1 hostport 9999\" -m udp -p udp -j DNAT --to-destination 10.1.1.2:443": true,
`COMMIT`: true,
}
for _, line := range lines {
t.Logf("Line: %s", line)
if len(strings.TrimSpace(line)) > 0 {
_, ok := expectedLines[strings.TrimSpace(line)]
assert.EqualValues(t, true, ok)
@ -362,7 +507,10 @@ func TestHostportManager(t *testing.T) {
remainingChains[strings.TrimSpace(line)] = true
}
}
expectDeletedChains := []string{"KUBE-HP-4YVONL46AKYWSKS3", "KUBE-HP-7THKRFSEH4GIIXK7", "KUBE-HP-5N7UH5JAXCVP5UJR"}
expectDeletedChains := []string{
"KUBE-HP-4YVONL46AKYWSKS3", "KUBE-HP-7THKRFSEH4GIIXK7", "KUBE-HP-5N7UH5JAXCVP5UJR",
"KUBE-HP-TUKTZ736U5JD5UTK", "KUBE-HP-CAAJ45HDITK7ARGM", "KUBE-HP-WFUNFVXVDLD5ZVXN", "KUBE-HP-4MFWH2F2NAOMYD6A",
}
for _, chain := range expectDeletedChains {
_, ok := remainingChains[chain]
assert.EqualValues(t, false, ok)
@ -372,6 +520,8 @@ func TestHostportManager(t *testing.T) {
for _, port := range portOpener.mem {
assert.EqualValues(t, true, port.closed)
}
// Clear all elements in hostPortMap
assert.Zero(t, len(manager.hostPortMap))
}
func TestGetHostportChain(t *testing.T) {
@ -397,7 +547,6 @@ func TestHostportManagerIPv6(t *testing.T) {
portOpener: portOpener.openFakeSocket,
execer: exec.New(),
}
testCases := []struct {
mapping *PodPortMapping
expectError bool
@ -499,7 +648,7 @@ func TestHostportManagerIPv6(t *testing.T) {
}
// Check port opened
expectedPorts := []hostport{{8080, "tcp"}, {8081, "udp"}, {8443, "tcp"}}
expectedPorts := []hostport{{IPv6, "", 8080, "tcp"}, {IPv6, "", 8081, "udp"}, {IPv6, "", 8443, "tcp"}}
openedPorts := make(map[hostport]bool)
for hp, port := range portOpener.mem {
if !port.closed {
@ -517,7 +666,7 @@ func TestHostportManagerIPv6(t *testing.T) {
err := iptables.SaveInto(utiliptables.TableNAT, raw)
assert.NoError(t, err)
lines := strings.Split(string(raw.Bytes()), "\n")
lines := strings.Split(raw.String(), "\n")
expectedLines := map[string]bool{
`*nat`: true,
`:KUBE-HOSTPORTS - [0:0]`: true,
@ -564,7 +713,7 @@ func TestHostportManagerIPv6(t *testing.T) {
raw.Reset()
err = iptables.SaveInto(utiliptables.TableNAT, raw)
assert.NoError(t, err)
lines = strings.Split(string(raw.Bytes()), "\n")
lines = strings.Split(raw.String(), "\n")
remainingChains := make(map[string]bool)
for _, line := range lines {
if strings.HasPrefix(line, ":") {

13
pkg/kubelet/dockershim/network/hostport/hostport_test.go
View File

@ -27,14 +27,15 @@ import (
)
type fakeSocket struct {
closed bool
port int32
protocol string
closed bool
ip string
}
func (f *fakeSocket) Close() error {
if f.closed {
return fmt.Errorf("Socket %q.%s already closed!", f.port, f.protocol)
return fmt.Errorf("socket %q.%s already closed", f.port, f.protocol)
}
f.closed = true
return nil
@ -52,7 +53,12 @@ func (f *fakeSocketManager) openFakeSocket(hp *hostport) (closeable, error) {
if socket, ok := f.mem[*hp]; ok && !socket.closed {
return nil, fmt.Errorf("hostport is occupied")
}
fs := &fakeSocket{hp.port, hp.protocol, false}
fs := &fakeSocket{
port: hp.port,
protocol: hp.protocol,
closed: false,
ip: hp.ip,
}
f.mem[*hp] = fs
return fs, nil
}
@ -80,5 +86,4 @@ func TestEnsureKubeHostportChains(t *testing.T) {
assert.EqualValues(t, len(chain.rules), 1)
assert.Contains(t, chain.rules[0], jumpRule)
}
}