From 43c3ae1bdc060198409cf87dc57a987a38c930ce Mon Sep 17 00:00:00 2001 From: Nimrod Gilboa Markevich Date: Wed, 25 May 2022 00:25:17 +0300 Subject: [PATCH] tmp: Periodically save tapper memory profile --- agent/go.mod | 1 + agent/go.sum | 2 ++ performance_analysis/run_tapper_benchmark.sh | 1 + tap/go.mod | 1 + tap/go.sum | 5 ++++- tap/passive_tapper.go | 20 ++++++++++++++++---- 6 files changed, 25 insertions(+), 5 deletions(-) diff --git a/agent/go.mod b/agent/go.mod index 7074cf75a..1685bd546 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -100,6 +100,7 @@ require ( github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pkg/profile v1.6.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday v1.6.0 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect diff --git a/agent/go.sum b/agent/go.sum index a080e07bf..fe8388658 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -574,6 +574,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.6.0 h1:hUDfIISABYI59DyeB3OTay/HxSRwTQ8rB/H83k6r5dM= +github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/performance_analysis/run_tapper_benchmark.sh b/performance_analysis/run_tapper_benchmark.sh index d2b7e6ccd..b27ceff0c 100755 --- a/performance_analysis/run_tapper_benchmark.sh +++ b/performance_analysis/run_tapper_benchmark.sh @@ -29,6 +29,7 @@ function run_single_bench() { else tapper_args+=("-i" "lo") fi + tapper_args+=("-tap.memprofile" "$MIZU_BENCHMARK_OUTPUT_DIR/${mode_num}_${mode_str}_${i}") nohup ./agent/build/mizuagent ${tapper_args[@]} > tapper.log 2>&1 & log " $i: Running client (hey)" diff --git a/tap/go.mod b/tap/go.mod index 67b1c3174..9cfdae303 100644 --- a/tap/go.mod +++ b/tap/go.mod @@ -7,6 +7,7 @@ require ( github.com/go-errors/errors v1.4.2 github.com/google/gopacket v1.1.19 github.com/hashicorp/golang-lru v0.5.4 + github.com/pkg/profile v1.6.0 github.com/shirou/gopsutil v3.21.11+incompatible github.com/struCoder/pidusage v0.2.1 github.com/up9inc/mizu/logger v0.0.0 diff --git a/tap/go.sum b/tap/go.sum index 786480904..74a36403b 100644 --- a/tap/go.sum +++ b/tap/go.sum @@ -114,6 +114,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.6.0 h1:hUDfIISABYI59DyeB3OTay/HxSRwTQ8rB/H83k6r5dM= +github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -290,5 +292,6 @@ sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/tap/passive_tapper.go b/tap/passive_tapper.go index 20d15a8a8..48b1cf350 100644 --- a/tap/passive_tapper.go +++ b/tap/passive_tapper.go @@ -18,6 +18,7 @@ import ( "time" "strconv" + "github.com/pkg/profile" "github.com/shirou/gopsutil/cpu" "github.com/struCoder/pidusage" "github.com/up9inc/mizu/logger" @@ -56,7 +57,7 @@ var staleTimeoutSeconds = flag.Int("staletimout", 120, "Max time in seconds to k var servicemesh = flag.Bool("servicemesh", false, "Record decrypted traffic if the cluster is configured with a service mesh and with mtls") var tls = flag.Bool("tls", false, "Enable TLS tapper") -var memprofile = flag.String("memprofile", "", "Write memory profile") +var memprofile = flag.String("tap.memprofile", "", "Write memory profile") type TapOpts struct { HostMode bool @@ -71,6 +72,17 @@ var mainPacketInputChan chan source.TcpPacketInfo // global var tlsTapperInstance *tlstapper.TlsTapper // global func StartPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem, extensionsRef []*api.Extension, options *api.TrafficFilteringOptions) { + if *memprofile != "" { + go func() { + for { + profilePath := profile.ProfilePath(*memprofile + "/" + time.Now().Format("2006-01-02T15:04:05-07:00")) + profile.Start(profile.MemProfileHeap, profilePath, profile.NoShutdownHook).Stop() + logger.Log.Info("Wrote Mem profiling") + <-time.After(time.Second * 60) + } + }() + } + extensions = extensionsRef filteringOptions = options @@ -248,9 +260,9 @@ func startPassiveTapper(streamsMap api.TcpStreamMap, assembler *tcpAssembler) { assembler.dumpStreamPool() } - if err := diagnose.DumpMemoryProfile(*memprofile); err != nil { - logger.Log.Errorf("Error dumping memory profile %v", err) - } + // if err := diagnose.DumpMemoryProfile(*memprofile); err != nil { + // logger.Log.Errorf("Error dumping memory profile %v", err) + // } assembler.waitAndDump()