Merge pull request #46033 from wojtek-t/reduce_memory_allocations_in_kube_proxy

Automatic merge from submit-queue

Reduce memory allocations in kube proxy

Memory allocation (and Go GarbageCollection) seems to be one of the most expensive operations in kube-proxy (I've seen profiles where it was more than 50%).

The commits are mostly independent from each other and all of them are mostly about reusing already allocated memory.

This PR is reducing memory allocation by ~5x (results below from 100-node load test):

before:
```
(pprof) top
38.64GB of 39.11GB total (98.79%)
Dropped 249 nodes (cum <= 0.20GB)
Showing top 10 nodes out of 61 (cum >= 0.20GB)
      flat  flat%   sum%        cum   cum%
   15.10GB 38.62% 38.62%    15.10GB 38.62%  bytes.makeSlice
    9.48GB 24.25% 62.87%     9.48GB 24.25%  runtime.rawstringtmp
    8.30GB 21.21% 84.07%    32.47GB 83.02%  k8s.io/kubernetes/pkg/proxy/iptables.(*Proxier).syncProxyRules
    2.08GB  5.31% 89.38%     2.08GB  5.31%  fmt.(*fmt).padString
    1.90GB  4.86% 94.24%     3.82GB  9.77%  strings.Join
    0.67GB  1.72% 95.96%     0.67GB  1.72%  runtime.hashGrow
    0.36GB  0.92% 96.88%     0.36GB  0.92%  runtime.stringtoslicebyte
    0.31GB  0.79% 97.67%     0.62GB  1.58%  encoding/base32.(*Encoding).EncodeToString
    0.24GB  0.62% 98.29%     0.24GB  0.62%  strings.genSplit
    0.20GB   0.5% 98.79%     0.20GB   0.5%  runtime.convT2E
```

after:
```
7.94GB of 8.13GB total (97.75%)
Dropped 311 nodes (cum <= 0.04GB)
Showing top 10 nodes out of 65 (cum >= 0.11GB)
      flat  flat%   sum%        cum   cum%
    3.32GB 40.87% 40.87%     8.05GB 99.05%  k8s.io/kubernetes/pkg/proxy/iptables.(*Proxier).syncProxyRules
    2.85GB 35.09% 75.95%     2.85GB 35.09%  runtime.rawstringtmp
    0.60GB  7.41% 83.37%     0.60GB  7.41%  runtime.hashGrow
    0.31GB  3.76% 87.13%     0.31GB  3.76%  runtime.stringtoslicebyte
    0.28GB  3.43% 90.56%     0.55GB  6.80%  encoding/base32.(*Encoding).EncodeToString
    0.19GB  2.29% 92.85%     0.19GB  2.29%  strings.genSplit
    0.18GB  2.17% 95.03%     0.18GB  2.17%  runtime.convT2E
    0.10GB  1.28% 96.31%     0.71GB  8.71%  runtime.mapassign
    0.10GB  1.21% 97.51%     0.10GB  1.21%  syscall.ByteSliceFromString
    0.02GB  0.23% 97.75%     0.11GB  1.38%  syscall.SlicePtrFromStrings
```

pkg/util/exec/exec.go

@@ -41,6 +41,8 @@ type Interface interface {
 // As more functionality is needed, this can grow.  Since Cmd is a struct, we will have
 // to replace fields with get/set method pairs.
 type Cmd interface {
+	// Run runs the command to the completion.
+	Run() error
 	// CombinedOutput runs the command and returns its combined standard output
 	// and standard error.  This follows the pattern of package os/exec.
 	CombinedOutput() ([]byte, error)
@@ -49,6 +51,7 @@ type Cmd interface {
 	SetDir(dir string)
 	SetStdin(in io.Reader)
 	SetStdout(out io.Writer)
+	SetStderr(out io.Writer)
 	// Stops the command by sending SIGTERM. It is not guaranteed the
 	// process will stop before this function returns. If the process is not
 	// responding, an internal timer function will send a SIGKILL to force
@@ -99,6 +102,15 @@ func (cmd *cmdWrapper) SetStdout(out io.Writer) {
 	cmd.Stdout = out
 }
 
+func (cmd *cmdWrapper) SetStderr(out io.Writer) {
+	cmd.Stderr = out
+}
+
+// Run is part of the Cmd interface.
+func (cmd *cmdWrapper) Run() error {
+	return (*osexec.Cmd)(cmd).Run()
+}
+
 // CombinedOutput is part of the Cmd interface.
 func (cmd *cmdWrapper) CombinedOutput() ([]byte, error) {
 	out, err := (*osexec.Cmd)(cmd).CombinedOutput()

pkg/util/exec/fake_exec.go

@@ -52,6 +52,7 @@ type FakeCmd struct {
 	Dirs                 []string
 	Stdin                io.Reader
 	Stdout               io.Writer
+	Stderr               io.Writer
 }
 
 func InitFakeCmd(fake *FakeCmd, cmd string, args ...string) Cmd {
@@ -73,6 +74,14 @@ func (fake *FakeCmd) SetStdout(out io.Writer) {
 	fake.Stdout = out
 }
 
+func (fake *FakeCmd) SetStderr(out io.Writer) {
+	fake.Stderr = out
+}
+
+func (fake *FakeCmd) Run() error {
+	return fmt.Errorf("unimplemented")
+}
+
 func (fake *FakeCmd) CombinedOutput() ([]byte, error) {
 	if fake.CombinedOutputCalls > len(fake.CombinedOutputScript)-1 {
 		panic("ran out of CombinedOutput() actions")

pkg/util/iptables/iptables.go

@@ -56,8 +56,8 @@ type Interface interface {
 	IsIpv6() bool
 	// Save calls `iptables-save` for table.
 	Save(table Table) ([]byte, error)
-	// SaveAll calls `iptables-save`.
-	SaveAll() ([]byte, error)
+	// SaveInto calls `iptables-save` for table and stores result in a given buffer.
+	SaveInto(table Table, buffer *bytes.Buffer) error
 	// Restore runs `iptables-restore` passing data through []byte.
 	// table is the Table to restore
 	// data should be formatted like the output of Save()
@@ -317,14 +317,23 @@ func (runner *runner) Save(table Table) ([]byte, error) {
 	return runner.exec.Command(cmdIPTablesSave, args...).CombinedOutput()
 }
 
-// SaveAll is part of Interface.
-func (runner *runner) SaveAll() ([]byte, error) {
+// SaveInto is part of Interface.
+func (runner *runner) SaveInto(table Table, buffer *bytes.Buffer) error {
 	runner.mu.Lock()
 	defer runner.mu.Unlock()
 
 	// run and return
-	glog.V(4).Infof("running iptables-save")
-	return runner.exec.Command(cmdIPTablesSave, []string{}...).CombinedOutput()
+	args := []string{"-t", string(table)}
+	glog.V(4).Infof("running iptables-save %v", args)
+	cmd := runner.exec.Command(cmdIPTablesSave, args...)
+	// Since CombinedOutput() doesn't support redirecting it to a buffer,
+	// we need to workaround it by redirecting stdout and stderr to buffer
+	// and explicitly calling Run() [CombinedOutput() underneath itself
+	// creates a new buffer, redirects stdout and stderr to it and also
+	// calls Run()].
+	cmd.SetStdout(buffer)
+	cmd.SetStderr(buffer)
+	return cmd.Run()
 }
 
 // Restore is part of Interface.
@@ -393,7 +402,7 @@ func (runner *runner) run(op operation, args []string) ([]byte, error) {
 
 	fullArgs := append(runner.waitFlag, string(op))
 	fullArgs = append(fullArgs, args...)
-	glog.V(4).Infof("running iptables %s %v", string(op), args)
+	glog.V(5).Infof("running iptables %s %v", string(op), args)
 	return runner.exec.Command(iptablesCmd, fullArgs...).CombinedOutput()
 	// Don't log err here - callers might not think it is an error.
 }

pkg/util/iptables/iptables_test.go

@@ -884,59 +884,6 @@ COMMIT
 	}
 }
 
-func TestSaveAll(t *testing.T) {
-	output := `# Generated by iptables-save v1.6.0 on Thu Jan 19 11:38:09 2017
-*filter
-:INPUT ACCEPT [15079:38410730]
-:FORWARD ACCEPT [0:0]
-:OUTPUT ACCEPT [11045:521562]
-COMMIT
-# Completed on Thu Jan 19 11:38:09 2017`
-
-	fcmd := exec.FakeCmd{
-		CombinedOutputScript: []exec.FakeCombinedOutputAction{
-			// iptables version check
-			func() ([]byte, error) { return []byte("iptables v1.9.22"), nil },
-			// iptables-restore version check
-			func() ([]byte, error) { return []byte("iptables-restore v1.9.22"), nil },
-			func() ([]byte, error) { return []byte(output), nil },
-			func() ([]byte, error) { return nil, &exec.FakeExitError{Status: 1} },
-		},
-	}
-	fexec := exec.FakeExec{
-		CommandScript: []exec.FakeCommandAction{
-			func(cmd string, args ...string) exec.Cmd { return exec.InitFakeCmd(&fcmd, cmd, args...) },
-			func(cmd string, args ...string) exec.Cmd { return exec.InitFakeCmd(&fcmd, cmd, args...) },
-			func(cmd string, args ...string) exec.Cmd { return exec.InitFakeCmd(&fcmd, cmd, args...) },
-			func(cmd string, args ...string) exec.Cmd { return exec.InitFakeCmd(&fcmd, cmd, args...) },
-		},
-	}
-	runner := New(&fexec, dbus.NewFake(nil, nil), ProtocolIpv4)
-	defer runner.Destroy()
-	// Success.
-	o, err := runner.SaveAll()
-	if err != nil {
-		t.Fatalf("expected success, got %v", err)
-	}
-
-	if string(o[:len(output)]) != output {
-		t.Errorf("expected output to be equal to mocked one, got %v", o)
-	}
-
-	if fcmd.CombinedOutputCalls != 3 {
-		t.Errorf("expected 3 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls)
-	}
-	if !sets.NewString(fcmd.CombinedOutputLog[2]...).HasAll("iptables-save") {
-		t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[2])
-	}
-
-	// Failure.
-	_, err = runner.SaveAll()
-	if err == nil {
-		t.Errorf("expected failure")
-	}
-}
-
 func TestRestore(t *testing.T) {
 	fcmd := exec.FakeCmd{
 		CombinedOutputScript: []exec.FakeCombinedOutputAction{

pkg/util/iptables/testing/fake.go

@@ -17,6 +17,7 @@ limitations under the License.
 package testing
 
 import (
+	"bytes"
 	"fmt"
 	"strings"
 
@@ -78,8 +79,9 @@ func (f *FakeIPTables) Save(table iptables.Table) ([]byte, error) {
 	return lines, nil
 }
 
-func (*FakeIPTables) SaveAll() ([]byte, error) {
-	return make([]byte, 0), nil
+func (f *FakeIPTables) SaveInto(table iptables.Table, buffer *bytes.Buffer) error {
+	buffer.Write(f.Lines)
+	return nil
 }
 
 func (*FakeIPTables) Restore(table iptables.Table, data []byte, flush iptables.FlushFlag, counters iptables.RestoreCountersFlag) error {