From 8fb365df32d7e59d25ddc2f382bf015dbc308763 Mon Sep 17 00:00:00 2001 From: wojtekt Date: Tue, 28 Aug 2018 15:18:58 +0200 Subject: [PATCH] Reduce amount of allocations in kube-proxy --- pkg/proxy/iptables/proxier.go | 20 ++++++++++---------- pkg/proxy/iptables/proxier_test.go | 1 + pkg/proxy/ipvs/proxier.go | 17 ++++++++--------- pkg/proxy/ipvs/proxier_test.go | 1 + 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 470916ce705..401755c9833 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -242,11 +242,12 @@ type Proxier struct { // The following buffers are used to reuse memory and avoid allocations // that are significantly impacting performance. - iptablesData *bytes.Buffer - filterChains *bytes.Buffer - filterRules *bytes.Buffer - natChains *bytes.Buffer - natRules *bytes.Buffer + iptablesData *bytes.Buffer + existingFilterChainsData *bytes.Buffer + filterChains *bytes.Buffer + filterRules *bytes.Buffer + natChains *bytes.Buffer + natRules *bytes.Buffer // endpointChainsNumber is the total amount of endpointChains across all // services that we will generate (it is computed at the beginning of @@ -340,6 +341,7 @@ func NewProxier(ipt utiliptables.Interface, healthzServer: healthzServer, precomputedProbabilities: make([]string, 0, 1001), iptablesData: bytes.NewBuffer(nil), + existingFilterChainsData: bytes.NewBuffer(nil), filterChains: bytes.NewBuffer(nil), filterRules: bytes.NewBuffer(nil), natChains: bytes.NewBuffer(nil), @@ -682,14 +684,12 @@ func (proxier *Proxier) syncProxyRules() { // Get iptables-save output so we can check for existing chains and rules. // This will be a map of chain name to chain with rules as stored in iptables-save/iptables-restore existingFilterChains := make(map[utiliptables.Chain][]byte) - // TODO: Filter table is small so we're not reusing this buffer over rounds. - // However, to optimize it further, we should do that. - existingFilterChainsData := bytes.NewBuffer(nil) - err := proxier.iptables.SaveInto(utiliptables.TableFilter, existingFilterChainsData) + proxier.existingFilterChainsData.Reset() + err := proxier.iptables.SaveInto(utiliptables.TableFilter, proxier.existingFilterChainsData) if err != nil { // if we failed to get any rules glog.Errorf("Failed to execute iptables-save, syncing all rules: %v", err) } else { // otherwise parse the output - existingFilterChains = utiliptables.GetChainLines(utiliptables.TableFilter, existingFilterChainsData.Bytes()) + existingFilterChains = utiliptables.GetChainLines(utiliptables.TableFilter, proxier.existingFilterChainsData.Bytes()) } // IMPORTANT: existingNATChains may share memory with proxier.iptablesData. diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index c11cde9f0b1..f66683f2119 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -373,6 +373,7 @@ func NewFakeProxier(ipt utiliptables.Interface) *Proxier { healthChecker: newFakeHealthChecker(), precomputedProbabilities: make([]string, 0, 1001), iptablesData: bytes.NewBuffer(nil), + existingFilterChainsData: bytes.NewBuffer(nil), filterChains: bytes.NewBuffer(nil), filterRules: bytes.NewBuffer(nil), natChains: bytes.NewBuffer(nil), diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 35a65101815..9f4ffd2c44a 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -213,11 +213,12 @@ type Proxier struct { ipGetter IPGetter // The following buffers are used to reuse memory and avoid allocations // that are significantly impacting performance. - iptablesData *bytes.Buffer - natChains *bytes.Buffer - filterChains *bytes.Buffer - natRules *bytes.Buffer - filterRules *bytes.Buffer + iptablesData *bytes.Buffer + filterChainsData *bytes.Buffer + natChains *bytes.Buffer + filterChains *bytes.Buffer + natRules *bytes.Buffer + filterRules *bytes.Buffer // Added as a member to the struct to allow injection for testing. netlinkHandle NetLinkHandle // ipsetList is the list of ipsets that ipvs proxier used. @@ -365,6 +366,7 @@ func NewProxier(ipt utiliptables.Interface, ipvsScheduler: scheduler, ipGetter: &realIPGetter{nl: NewNetLinkHandle()}, iptablesData: bytes.NewBuffer(nil), + filterChainsData: bytes.NewBuffer(nil), natChains: bytes.NewBuffer(nil), natRules: bytes.NewBuffer(nil), filterChains: bytes.NewBuffer(nil), @@ -1345,10 +1347,7 @@ func (proxier *Proxier) acceptIPVSTraffic() { // createAndLinkeKubeChain create all kube chains that ipvs proxier need and write basic link. func (proxier *Proxier) createAndLinkeKubeChain() { - // TODO: Filter table is small so we're not reusing this buffer over rounds. - // However, to optimize it further, we should do that. - filterBuffer := bytes.NewBuffer(nil) - existingFilterChains := proxier.getExistingChains(filterBuffer, utiliptables.TableFilter) + existingFilterChains := proxier.getExistingChains(proxier.filterChainsData, utiliptables.TableFilter) existingNATChains := proxier.getExistingChains(proxier.iptablesData, utiliptables.TableNAT) // Make sure we keep stats for the top-level chains diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index 4f2ea198b7f..85299f4667d 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -163,6 +163,7 @@ func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset u ipvsScheduler: DefaultScheduler, ipGetter: &fakeIPGetter{nodeIPs: nodeIPs}, iptablesData: bytes.NewBuffer(nil), + filterChainsData: bytes.NewBuffer(nil), natChains: bytes.NewBuffer(nil), natRules: bytes.NewBuffer(nil), filterChains: bytes.NewBuffer(nil),