diff --git a/pkg/kubelet/dockertools/docker_test.go b/pkg/kubelet/dockertools/docker_test.go index 25825b52291..5e0804ec9cd 100644 --- a/pkg/kubelet/dockertools/docker_test.go +++ b/pkg/kubelet/dockertools/docker_test.go @@ -651,7 +651,7 @@ func TestFindContainersByPod(t *testing.T) { }, } fakeClient := NewFakeDockerClient() - np, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone) + np, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8") // image back-off is set to nil, this test should not pull images containerManager := NewFakeDockerManager(fakeClient, &record.FakeRecorder{}, nil, nil, &cadvisorapi.MachineInfo{}, options.GetDefaultPodInfraContainerImage(), 0, 0, "", &containertest.FakeOS{}, np, nil, nil, nil) for i, test := range tests { diff --git a/pkg/kubelet/dockertools/manager_test.go b/pkg/kubelet/dockertools/manager_test.go index 63c3bca13ea..74417887ed6 100644 --- a/pkg/kubelet/dockertools/manager_test.go +++ b/pkg/kubelet/dockertools/manager_test.go @@ -107,7 +107,7 @@ func createTestDockerManager(fakeHTTPClient *fakeHTTP, fakeDocker *FakeDockerCli } fakeRecorder := &record.FakeRecorder{} containerRefManager := kubecontainer.NewRefManager() - networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone) + networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8") dockerManager := NewFakeDockerManager( fakeDocker, fakeRecorder, diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index c60eade7061..a9c962b0a21 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -385,7 +385,7 @@ func NewMainKubelet( } glog.Infof("Hairpin mode set to %q", klet.hairpinMode) - if plug, err := network.InitNetworkPlugin(networkPlugins, networkPluginName, &networkHost{klet}, klet.hairpinMode); err != nil { + if plug, err := network.InitNetworkPlugin(networkPlugins, networkPluginName, &networkHost{klet}, klet.hairpinMode, klet.nonMasqueradeCIDR); err != nil { return nil, err } else { klet.networkPlugin = plug diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 347fcd801bc..b40ad7defad 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -188,7 +188,7 @@ func newTestKubeletWithImageList(t *testing.T, imageList []kubecontainer.Image) kubelet.nodeName = testKubeletHostname kubelet.runtimeState = newRuntimeState(maxWaitForContainerRuntime) kubelet.runtimeState.setNetworkState(nil) - kubelet.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone) + kubelet.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, kubelet.nonMasqueradeCIDR) if tempDir, err := ioutil.TempDir("/tmp", "kubelet_test."); err != nil { t.Fatalf("can't make a temp rootdir: %v", err) } else { diff --git a/pkg/kubelet/network/cni/cni.go b/pkg/kubelet/network/cni/cni.go index 5eac8772fb9..dc0be91acce 100644 --- a/pkg/kubelet/network/cni/cni.go +++ b/pkg/kubelet/network/cni/cni.go @@ -94,7 +94,7 @@ func getDefaultCNINetwork(pluginDir, vendorCNIDirPrefix string) (*cniNetwork, er return nil, fmt.Errorf("No valid networks found in %s", pluginDir) } -func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error { +func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error { plugin.host = host return nil } diff --git a/pkg/kubelet/network/cni/cni_test.go b/pkg/kubelet/network/cni/cni_test.go index 32beb984898..fe4aedee846 100644 --- a/pkg/kubelet/network/cni/cni_test.go +++ b/pkg/kubelet/network/cni/cni_test.go @@ -145,7 +145,7 @@ func newTestDockerManager() (*dockertools.DockerManager, *dockertools.FakeDocker fakeDocker := dockertools.NewFakeDockerClient() fakeRecorder := &record.FakeRecorder{} containerRefManager := kubecontainer.NewRefManager() - networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone) + networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8") dockerManager := dockertools.NewFakeDockerManager( fakeDocker, fakeRecorder, @@ -175,7 +175,7 @@ func TestCNIPlugin(t *testing.T) { installPluginUnderTest(t, testVendorCNIDirPrefix, testNetworkConfigPath, vendorName, pluginName) np := probeNetworkPluginsWithVendorCNIDirPrefix(path.Join(testNetworkConfigPath, pluginName), testVendorCNIDirPrefix) - plug, err := network.InitNetworkPlugin(np, "cni", NewFakeHost(nil), componentconfig.HairpinNone) + plug, err := network.InitNetworkPlugin(np, "cni", NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8") if err != nil { t.Fatalf("Failed to select the desired plugin: %v", err) } diff --git a/pkg/kubelet/network/exec/exec.go b/pkg/kubelet/network/exec/exec.go index 685da4b3611..83d934b57ce 100644 --- a/pkg/kubelet/network/exec/exec.go +++ b/pkg/kubelet/network/exec/exec.go @@ -105,7 +105,7 @@ func ProbeNetworkPlugins(pluginDir string) []network.NetworkPlugin { return execPlugins } -func (plugin *execNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error { +func (plugin *execNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error { err := plugin.validate() if err != nil { return err diff --git a/pkg/kubelet/network/exec/exec_test.go b/pkg/kubelet/network/exec/exec_test.go index 75fa0c985d7..4a682864d12 100644 --- a/pkg/kubelet/network/exec/exec_test.go +++ b/pkg/kubelet/network/exec/exec_test.go @@ -135,7 +135,7 @@ func TestSelectPlugin(t *testing.T) { installPluginUnderTest(t, "", testPluginPath, pluginName, nil) - plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone) + plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8") if err != nil { t.Errorf("Failed to select the desired plugin: %v", err) } @@ -157,7 +157,7 @@ func TestSelectVendoredPlugin(t *testing.T) { installPluginUnderTest(t, vendor, testPluginPath, pluginName, nil) vendoredPluginName := fmt.Sprintf("%s/%s", vendor, pluginName) - plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), vendoredPluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone) + plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), vendoredPluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8") if err != nil { t.Errorf("Failed to select the desired plugin: %v", err) } @@ -178,7 +178,7 @@ func TestSelectWrongPlugin(t *testing.T) { installPluginUnderTest(t, "", testPluginPath, pluginName, nil) wrongPlugin := "abcd" - plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), wrongPlugin, nettest.NewFakeHost(nil), componentconfig.HairpinNone) + plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), wrongPlugin, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8") if plug != nil || err == nil { t.Errorf("Expected to see an error. Wrong plugin selected.") } @@ -206,7 +206,7 @@ func TestPluginValidation(t *testing.T) { } f.Close() - _, err = network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone) + _, err = network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8") if err == nil { // we expected an error here because validation would have failed t.Errorf("Expected non-nil value.") @@ -224,7 +224,7 @@ func TestPluginSetupHook(t *testing.T) { installPluginUnderTest(t, "", testPluginPath, pluginName, nil) - plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone) + plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8") err = plug.SetUpPod("podNamespace", "podName", kubecontainer.ContainerID{Type: "docker", ID: "dockerid2345"}) if err != nil { @@ -252,7 +252,7 @@ func TestPluginTearDownHook(t *testing.T) { installPluginUnderTest(t, "", testPluginPath, pluginName, nil) - plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone) + plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8") err = plug.TearDownPod("podNamespace", "podName", kubecontainer.ContainerID{Type: "docker", ID: "dockerid2345"}) if err != nil { @@ -280,7 +280,7 @@ func TestPluginStatusHook(t *testing.T) { installPluginUnderTest(t, "", testPluginPath, pluginName, nil) - plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone) + plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8") ip, err := plug.GetPodNetworkStatus("namespace", "name", kubecontainer.ContainerID{Type: "docker", ID: "dockerid2345"}) if err != nil { @@ -316,7 +316,7 @@ func TestPluginStatusHookIPv6(t *testing.T) { } installPluginUnderTest(t, "", testPluginPath, pluginName, execTemplate) - plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone) + plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8") if err != nil { t.Errorf("InitNetworkPlugin() failed: %v", err) } diff --git a/pkg/kubelet/network/kubenet/kubenet_linux.go b/pkg/kubelet/network/kubenet/kubenet_linux.go index 785e3fd21b1..0461606f2de 100644 --- a/pkg/kubelet/network/kubenet/kubenet_linux.go +++ b/pkg/kubelet/network/kubenet/kubenet_linux.go @@ -79,7 +79,8 @@ type kubenetNetworkPlugin struct { iptables utiliptables.Interface // vendorDir is passed by kubelet network-plugin-dir parameter. // kubenet will search for cni binaries in DefaultCNIDir first, then continue to vendorDir. - vendorDir string + vendorDir string + nonMasqueradeCIDR string } func NewPlugin(networkPluginDir string) network.NetworkPlugin { @@ -88,16 +89,17 @@ func NewPlugin(networkPluginDir string) network.NetworkPlugin { dbus := utildbus.New() iptInterface := utiliptables.New(execer, dbus, protocol) return &kubenetNetworkPlugin{ - podIPs: make(map[kubecontainer.ContainerID]string), - hostPortMap: make(map[hostport]closeable), - MTU: 1460, //TODO: don't hardcode this - execer: utilexec.New(), - iptables: iptInterface, - vendorDir: networkPluginDir, + podIPs: make(map[kubecontainer.ContainerID]string), + hostPortMap: make(map[hostport]closeable), + MTU: 1460, //TODO: don't hardcode this + execer: utilexec.New(), + iptables: iptInterface, + vendorDir: networkPluginDir, + nonMasqueradeCIDR: "10.0.0.0/8", } } -func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error { +func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error { plugin.host = host plugin.hairpinMode = hairpinMode plugin.cniConfig = &libcni.CNIConfig{ @@ -132,6 +134,23 @@ func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componen return fmt.Errorf("Failed to generate loopback config: %v", err) } + plugin.nonMasqueradeCIDR = nonMasqueradeCIDR + // Need to SNAT outbound traffic from cluster + if err = plugin.ensureMasqRule(); err != nil { + return err + } + return nil +} + +// TODO: move thic logic into cni bridge plugin and remove this from kubenet +func (plugin *kubenetNetworkPlugin) ensureMasqRule() error { + if _, err := plugin.iptables.EnsureRule(utiliptables.Append, utiliptables.TableNAT, utiliptables.ChainPostrouting, + "-m", "comment", "--comment", "kubenet: SNAT for outbound traffic from cluster", + "-m", "addrtype", "!", "--dst-type", "LOCAL", + "!", "-d", plugin.nonMasqueradeCIDR, + "-j", "MASQUERADE"); err != nil { + return fmt.Errorf("Failed to ensure that %s chain %s jumps to MASQUERADE: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, err) + } return nil } @@ -167,7 +186,7 @@ const NET_CONFIG_TEMPLATE = `{ "mtu": %d, "addIf": "%s", "isGateway": true, - "ipMasq": true, + "ipMasq": false, "ipam": { "type": "host-local", "subnet": "%s", @@ -354,6 +373,11 @@ func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id k } plugin.syncHostportsRules() + + // Need to SNAT outbound traffic from cluster + if err = plugin.ensureMasqRule(); err != nil { + glog.Errorf("Failed to ensure MASQ rule: %v", err) + } return nil } @@ -391,6 +415,11 @@ func (plugin *kubenetNetworkPlugin) TearDownPod(namespace string, name string, i delete(plugin.podIPs, id) plugin.syncHostportsRules() + + // Need to SNAT outbound traffic from cluster + if err := plugin.ensureMasqRule(); err != nil { + glog.Errorf("Failed to ensure MASQ rule: %v", err) + } return nil } diff --git a/pkg/kubelet/network/kubenet/kubenet_unsupported.go b/pkg/kubelet/network/kubenet/kubenet_unsupported.go index e41eaeeb383..50ceb2af088 100644 --- a/pkg/kubelet/network/kubenet/kubenet_unsupported.go +++ b/pkg/kubelet/network/kubenet/kubenet_unsupported.go @@ -34,7 +34,7 @@ func NewPlugin(networkPluginDir string) network.NetworkPlugin { return &kubenetNetworkPlugin{} } -func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error { +func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error { return fmt.Errorf("Kubenet is not supported in this build") } diff --git a/pkg/kubelet/network/mock_network/network_plugins.go b/pkg/kubelet/network/mock_network/network_plugins.go index 6055de6e218..9b93d7eebda 100644 --- a/pkg/kubelet/network/mock_network/network_plugins.go +++ b/pkg/kubelet/network/mock_network/network_plugins.go @@ -78,7 +78,7 @@ func (_mr *_MockNetworkPluginRecorder) GetPodNetworkStatus(arg0, arg1, arg2 inte return _mr.mock.ctrl.RecordCall(_mr.mock, "GetPodNetworkStatus", arg0, arg1, arg2) } -func (_m *MockNetworkPlugin) Init(_param0 network.Host, _param1 componentconfig.HairpinMode) error { +func (_m *MockNetworkPlugin) Init(_param0 network.Host, _param1 componentconfig.HairpinMode, nonMasqueradeCIDR string) error { ret := _m.ctrl.Call(_m, "Init", _param0, _param1) ret0, _ := ret[0].(error) return ret0 diff --git a/pkg/kubelet/network/plugins.go b/pkg/kubelet/network/plugins.go index 1d52415b22c..405a2d1564c 100644 --- a/pkg/kubelet/network/plugins.go +++ b/pkg/kubelet/network/plugins.go @@ -52,7 +52,7 @@ const ( type NetworkPlugin interface { // Init initializes the plugin. This will be called exactly once // before any other methods are called. - Init(host Host, hairpinMode componentconfig.HairpinMode) error + Init(host Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error // Called on various events like: // NET_PLUGIN_EVENT_POD_CIDR_CHANGE @@ -105,11 +105,11 @@ type Host interface { } // InitNetworkPlugin inits the plugin that matches networkPluginName. Plugins must have unique names. -func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host Host, hairpinMode componentconfig.HairpinMode) (NetworkPlugin, error) { +func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) (NetworkPlugin, error) { if networkPluginName == "" { // default to the no_op plugin plug := &NoopNetworkPlugin{} - if err := plug.Init(host, hairpinMode); err != nil { + if err := plug.Init(host, hairpinMode, nonMasqueradeCIDR); err != nil { return nil, err } return plug, nil @@ -134,7 +134,7 @@ func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host H chosenPlugin := pluginMap[networkPluginName] if chosenPlugin != nil { - err := chosenPlugin.Init(host, hairpinMode) + err := chosenPlugin.Init(host, hairpinMode, nonMasqueradeCIDR) if err != nil { allErrs = append(allErrs, fmt.Errorf("Network plugin %q failed init: %v", networkPluginName, err)) } else { @@ -156,7 +156,7 @@ type NoopNetworkPlugin struct { const sysctlBridgeCallIptables = "net/bridge/bridge-nf-call-iptables" -func (plugin *NoopNetworkPlugin) Init(host Host, hairpinMode componentconfig.HairpinMode) error { +func (plugin *NoopNetworkPlugin) Init(host Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string) error { // Set bridge-nf-call-iptables=1 to maintain compatibility with older // kubernetes versions to ensure the iptables-based kube proxy functions // correctly. Other plugins are responsible for setting this correctly diff --git a/pkg/kubelet/network/plugins_test.go b/pkg/kubelet/network/plugins_test.go index e6ea9ed283f..a5baf93a3da 100644 --- a/pkg/kubelet/network/plugins_test.go +++ b/pkg/kubelet/network/plugins_test.go @@ -25,7 +25,7 @@ import ( func TestSelectDefaultPlugin(t *testing.T) { all_plugins := []NetworkPlugin{} - plug, err := InitNetworkPlugin(all_plugins, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone) + plug, err := InitNetworkPlugin(all_plugins, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, "10.0.0.0/8") if err != nil { t.Fatalf("Unexpected error in selecting default plugin: %v", err) } diff --git a/pkg/kubelet/runonce_test.go b/pkg/kubelet/runonce_test.go index e84e1e823ef..dda20a32fcf 100644 --- a/pkg/kubelet/runonce_test.go +++ b/pkg/kubelet/runonce_test.go @@ -84,7 +84,7 @@ func TestRunOnce(t *testing.T) { } kb.containerManager = cm.NewStubContainerManager() - kb.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone) + kb.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone, kb.nonMasqueradeCIDR) // TODO: Factor out "StatsProvider" from Kubelet so we don't have a cyclic dependency volumeStatsAggPeriod := time.Second * 10 kb.resourceAnalyzer = stats.NewResourceAnalyzer(kb, volumeStatsAggPeriod, kb.containerRuntime)