github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 19:56:01 +00:00
Code Issues Projects Releases Wiki Activity

kubenet masqurade for outbound traffic

Browse Source
This commit is contained in:
Minhan Xia 2016-06-09 10:32:28 -07:00
parent 3d332a047b
commit 1276a91638
14 changed files with 63 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/kubelet/dockertools/docker_test.go
View File

@ -651,7 +651,7 @@ func TestFindContainersByPod(t *testing.T) {
},
}
fakeClient := NewFakeDockerClient()
np, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
np, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
// image back-off is set to nil, this test should not pull images
containerManager := NewFakeDockerManager(fakeClient, &record.FakeRecorder{}, nil, nil, &cadvisorapi.MachineInfo{}, options.GetDefaultPodInfraContainerImage(), 0, 0, "", &containertest.FakeOS{}, np, nil, nil, nil)
for i, test := range tests {

2
pkg/kubelet/dockertools/manager_test.go
View File

@ -107,7 +107,7 @@ func createTestDockerManager(fakeHTTPClient *fakeHTTP, fakeDocker *FakeDockerCli
}
fakeRecorder := &record.FakeRecorder{}
containerRefManager := kubecontainer.NewRefManager()
networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
dockerManager := NewFakeDockerManager(
fakeDocker,
fakeRecorder,

2
pkg/kubelet/kubelet.go
View File

@ -385,7 +385,7 @@ func NewMainKubelet(
}
glog.Infof("Hairpin mode set to %q", klet.hairpinMode)
if plug, err := network.InitNetworkPlugin(networkPlugins, networkPluginName, &networkHost{klet}, klet.hairpinMode); err != nil {
if plug, err := network.InitNetworkPlugin(networkPlugins, networkPluginName, &networkHost{klet}, klet.hairpinMode, klet.nonMasqueradeCIDR); err != nil {
return nil, err
} else {
klet.networkPlugin = plug

2
pkg/kubelet/kubelet_test.go
View File

@ -188,7 +188,7 @@ func newTestKubeletWithImageList(t *testing.T, imageList []kubecontainer.Image)
kubelet.nodeName = testKubeletHostname
kubelet.runtimeState = newRuntimeState(maxWaitForContainerRuntime)
kubelet.runtimeState.setNetworkState(nil)
kubelet.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
kubelet.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, kubelet.nonMasqueradeCIDR)
if tempDir, err := ioutil.TempDir("/tmp", "kubelet_test."); err != nil {
t.Fatalf("can't make a temp rootdir: %v", err)
} else {

2
pkg/kubelet/network/cni/cni.go
View File

@ -94,7 +94,7 @@ func getDefaultCNINetwork(pluginDir, vendorCNIDirPrefix string) (*cniNetwork, er
return nil, fmt.Errorf("No valid networks found in %s", pluginDir)
}
func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error {
func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error {
plugin.host = host
return nil
}

4
pkg/kubelet/network/cni/cni_test.go
View File

@ -145,7 +145,7 @@ func newTestDockerManager() (*dockertools.DockerManager, *dockertools.FakeDocker
fakeDocker := dockertools.NewFakeDockerClient()
fakeRecorder := &record.FakeRecorder{}
containerRefManager := kubecontainer.NewRefManager()
networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
dockerManager := dockertools.NewFakeDockerManager(
fakeDocker,
fakeRecorder,
@ -175,7 +175,7 @@ func TestCNIPlugin(t *testing.T) {
installPluginUnderTest(t, testVendorCNIDirPrefix, testNetworkConfigPath, vendorName, pluginName)
np := probeNetworkPluginsWithVendorCNIDirPrefix(path.Join(testNetworkConfigPath, pluginName), testVendorCNIDirPrefix)
plug, err := network.InitNetworkPlugin(np, "cni", NewFakeHost(nil), componentconfig.HairpinNone)
plug, err := network.InitNetworkPlugin(np, "cni", NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
if err != nil {
t.Fatalf("Failed to select the desired plugin: %v", err)
}

2
pkg/kubelet/network/exec/exec.go
View File

@ -105,7 +105,7 @@ func ProbeNetworkPlugins(pluginDir string) []network.NetworkPlugin {
return execPlugins
}
func (plugin *execNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error {
func (plugin *execNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error {
err := plugin.validate()
if err != nil {
return err

16
pkg/kubelet/network/exec/exec_test.go
View File

@ -135,7 +135,7 @@ func TestSelectPlugin(t *testing.T) {
installPluginUnderTest(t, "", testPluginPath, pluginName, nil)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
if err != nil {
t.Errorf("Failed to select the desired plugin: %v", err)
}
@ -157,7 +157,7 @@ func TestSelectVendoredPlugin(t *testing.T) {
installPluginUnderTest(t, vendor, testPluginPath, pluginName, nil)
vendoredPluginName := fmt.Sprintf("%s/%s", vendor, pluginName)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), vendoredPluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), vendoredPluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
if err != nil {
t.Errorf("Failed to select the desired plugin: %v", err)
}
@ -178,7 +178,7 @@ func TestSelectWrongPlugin(t *testing.T) {
installPluginUnderTest(t, "", testPluginPath, pluginName, nil)
wrongPlugin := "abcd"
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), wrongPlugin, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), wrongPlugin, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
if plug != nil || err == nil {
t.Errorf("Expected to see an error. Wrong plugin selected.")
}
@ -206,7 +206,7 @@ func TestPluginValidation(t *testing.T) {
}
f.Close()
_, err = network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
_, err = network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
if err == nil {
// we expected an error here because validation would have failed
t.Errorf("Expected non-nil value.")
@ -224,7 +224,7 @@ func TestPluginSetupHook(t *testing.T) {
installPluginUnderTest(t, "", testPluginPath, pluginName, nil)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
err = plug.SetUpPod("podNamespace", "podName", kubecontainer.ContainerID{Type: "docker", ID: "dockerid2345"})
if err != nil {
@ -252,7 +252,7 @@ func TestPluginTearDownHook(t *testing.T) {
installPluginUnderTest(t, "", testPluginPath, pluginName, nil)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
err = plug.TearDownPod("podNamespace", "podName", kubecontainer.ContainerID{Type: "docker", ID: "dockerid2345"})
if err != nil {
@ -280,7 +280,7 @@ func TestPluginStatusHook(t *testing.T) {
installPluginUnderTest(t, "", testPluginPath, pluginName, nil)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
ip, err := plug.GetPodNetworkStatus("namespace", "name", kubecontainer.ContainerID{Type: "docker", ID: "dockerid2345"})
if err != nil {
@ -316,7 +316,7 @@ func TestPluginStatusHookIPv6(t *testing.T) {
}
installPluginUnderTest(t, "", testPluginPath, pluginName, execTemplate)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
if err != nil {
t.Errorf("InitNetworkPlugin() failed: %v", err)
}

47
pkg/kubelet/network/kubenet/kubenet_linux.go
View File

@ -79,7 +79,8 @@ type kubenetNetworkPlugin struct {
iptables utiliptables.Interface
// vendorDir is passed by kubelet network-plugin-dir parameter.
// kubenet will search for cni binaries in DefaultCNIDir first, then continue to vendorDir.
vendorDir string
vendorDir string
nonMasqueradeCIDR string
}
func NewPlugin(networkPluginDir string) network.NetworkPlugin {
@ -88,16 +89,17 @@ func NewPlugin(networkPluginDir string) network.NetworkPlugin {
dbus := utildbus.New()
iptInterface := utiliptables.New(execer, dbus, protocol)
return &kubenetNetworkPlugin{
podIPs: make(map[kubecontainer.ContainerID]string),
hostPortMap: make(map[hostport]closeable),
MTU: 1460, //TODO: don't hardcode this
execer: utilexec.New(),
iptables: iptInterface,
vendorDir: networkPluginDir,
podIPs: make(map[kubecontainer.ContainerID]string),
hostPortMap: make(map[hostport]closeable),
MTU: 1460, //TODO: don't hardcode this
execer: utilexec.New(),
iptables: iptInterface,
vendorDir: networkPluginDir,
nonMasqueradeCIDR: "10.0.0.0/8",
}
}
func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error {
func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error {
plugin.host = host
plugin.hairpinMode = hairpinMode
plugin.cniConfig = &libcni.CNIConfig{
@ -132,6 +134,23 @@ func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componen
return fmt.Errorf("Failed to generate loopback config: %v", err)
}
plugin.nonMasqueradeCIDR = nonMasqueradeCIDR
// Need to SNAT outbound traffic from cluster
if err = plugin.ensureMasqRule(); err != nil {
return err
}
return nil
}
// TODO: move thic logic into cni bridge plugin and remove this from kubenet
func (plugin *kubenetNetworkPlugin) ensureMasqRule() error {
if _, err := plugin.iptables.EnsureRule(utiliptables.Append, utiliptables.TableNAT, utiliptables.ChainPostrouting,
"-m", "comment", "--comment", "kubenet: SNAT for outbound traffic from cluster",
"-m", "addrtype", "!", "--dst-type", "LOCAL",
"!", "-d", plugin.nonMasqueradeCIDR,
"-j", "MASQUERADE"); err != nil {
return fmt.Errorf("Failed to ensure that %s chain %s jumps to MASQUERADE: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, err)
}
return nil
}
@ -167,7 +186,7 @@ const NET_CONFIG_TEMPLATE = `{
"mtu": %d,
"addIf": "%s",
"isGateway": true,
"ipMasq": true,
"ipMasq": false,
"ipam": {
"type": "host-local",
"subnet": "%s",
@ -354,6 +373,11 @@ func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id k
}
plugin.syncHostportsRules()
// Need to SNAT outbound traffic from cluster
if err = plugin.ensureMasqRule(); err != nil {
glog.Errorf("Failed to ensure MASQ rule: %v", err)
}
return nil
}
@ -391,6 +415,11 @@ func (plugin *kubenetNetworkPlugin) TearDownPod(namespace string, name string, i
delete(plugin.podIPs, id)
plugin.syncHostportsRules()
// Need to SNAT outbound traffic from cluster
if err := plugin.ensureMasqRule(); err != nil {
glog.Errorf("Failed to ensure MASQ rule: %v", err)
}
return nil
}

2
pkg/kubelet/network/kubenet/kubenet_unsupported.go
View File

@ -34,7 +34,7 @@ func NewPlugin(networkPluginDir string) network.NetworkPlugin {
return &kubenetNetworkPlugin{}
}
func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error {
func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error {
return fmt.Errorf("Kubenet is not supported in this build")
}

2
pkg/kubelet/network/mock_network/network_plugins.go
View File

@ -78,7 +78,7 @@ func (_mr *_MockNetworkPluginRecorder) GetPodNetworkStatus(arg0, arg1, arg2 inte
return _mr.mock.ctrl.RecordCall(_mr.mock, "GetPodNetworkStatus", arg0, arg1, arg2)
}
func (_m *MockNetworkPlugin) Init(_param0 network.Host, _param1 componentconfig.HairpinMode) error {
func (_m *MockNetworkPlugin) Init(_param0 network.Host, _param1 componentconfig.HairpinMode, nonMasqueradeCIDR string) error {
ret := _m.ctrl.Call(_m, "Init", _param0, _param1)
ret0, _ := ret[0].(error)
return ret0

10
pkg/kubelet/network/plugins.go
View File

@ -52,7 +52,7 @@ const (
type NetworkPlugin interface {
// Init initializes the plugin. This will be called exactly once
// before any other methods are called.
Init(host Host, hairpinMode componentconfig.HairpinMode) error
Init(host Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error
// Called on various events like:
// NET_PLUGIN_EVENT_POD_CIDR_CHANGE
@ -105,11 +105,11 @@ type Host interface {
}
// InitNetworkPlugin inits the plugin that matches networkPluginName. Plugins must have unique names.
func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host Host, hairpinMode componentconfig.HairpinMode) (NetworkPlugin, error) {
func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) (NetworkPlugin, error) {
if networkPluginName == "" {
// default to the no_op plugin
plug := &NoopNetworkPlugin{}
if err := plug.Init(host, hairpinMode); err != nil {
if err := plug.Init(host, hairpinMode, nonMasqueradeCIDR); err != nil {
return nil, err
}
return plug, nil
@ -134,7 +134,7 @@ func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host H
chosenPlugin := pluginMap[networkPluginName]
if chosenPlugin != nil {
err := chosenPlugin.Init(host, hairpinMode)
err := chosenPlugin.Init(host, hairpinMode, nonMasqueradeCIDR)
if err != nil {
allErrs = append(allErrs, fmt.Errorf("Network plugin %q failed init: %v", networkPluginName, err))
} else {
@ -156,7 +156,7 @@ type NoopNetworkPlugin struct {
const sysctlBridgeCallIptables = "net/bridge/bridge-nf-call-iptables"
func (plugin *NoopNetworkPlugin) Init(host Host, hairpinMode componentconfig.HairpinMode) error {
func (plugin *NoopNetworkPlugin) Init(host Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error {
// Set bridge-nf-call-iptables=1 to maintain compatibility with older
// kubernetes versions to ensure the iptables-based kube proxy functions
// correctly. Other plugins are responsible for setting this correctly

2
pkg/kubelet/network/plugins_test.go
View File

@ -25,7 +25,7 @@ import (
func TestSelectDefaultPlugin(t *testing.T) {
all_plugins := []NetworkPlugin{}
plug, err := InitNetworkPlugin(all_plugins, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
plug, err := InitNetworkPlugin(all_plugins, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8")
if err != nil {
t.Fatalf("Unexpected error in selecting default plugin: %v", err)
}

2
pkg/kubelet/runonce_test.go
View File

@ -84,7 +84,7 @@ func TestRunOnce(t *testing.T) {
}
kb.containerManager = cm.NewStubContainerManager()
kb.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
kb.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, kb.nonMasqueradeCIDR)
// TODO: Factor out "StatsProvider" from Kubelet so we don't have a cyclic dependency
volumeStatsAggPeriod := time.Second * 10
kb.resourceAnalyzer = stats.NewResourceAnalyzer(kb, volumeStatsAggPeriod, kb.containerRuntime)