From d684dee7a4ce2343a3c924467ed2bd40249663c0 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Mon, 12 Jul 2021 17:47:46 +0300 Subject: [PATCH 01/14] WIP --- api/go.mod | 1 + api/go.sum | 3 + api/pkg/api/main.go | 20 ++++++ api/pkg/database/main.go | 1 + api/pkg/database/size_enforcer.go | 94 ++++++++++++++++++++++++++++ api/pkg/models/models.go | 1 + cli/cmd/tapRunner.go | 2 +- {cli => shared}/debounce/debounce.go | 0 8 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 api/pkg/database/size_enforcer.go rename {cli => shared}/debounce/debounce.go (100%) diff --git a/api/go.mod b/api/go.mod index 9f8825215..27833d4e4 100644 --- a/api/go.mod +++ b/api/go.mod @@ -23,6 +23,7 @@ require ( k8s.io/api v0.21.0 k8s.io/apimachinery v0.21.0 k8s.io/client-go v0.21.0 + github.com/fsnotify/fsnotify v1.4.9 ) replace github.com/up9inc/mizu/shared v0.0.0 => ../shared diff --git a/api/go.sum b/api/go.sum index 855914a0f..baf901d28 100644 --- a/api/go.sum +++ b/api/go.sum @@ -73,6 +73,8 @@ github.com/fasthttp/websocket v1.4.3-beta.1/go.mod h1:JGrgLaT02bL9NuJkZbHN8mVV2t github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -411,6 +413,7 @@ golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/api/pkg/api/main.go b/api/pkg/api/main.go index 142f43700..10faec2ad 100644 --- a/api/pkg/api/main.go +++ b/api/pkg/api/main.go @@ -160,6 +160,7 @@ func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) { ResolvedDestination: resolvedDestination, IsOutgoing: connectionInfo.IsOutgoing, } + mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry) database.GetEntriesTable().Create(&mizuEntry) baseEntry := models.BaseEntryDetails{} @@ -179,3 +180,22 @@ func getServiceNameFromUrl(inputUrl string) (string, string) { func CheckIsServiceIP(address string) bool { return k8sResolver.CheckIsServiceIP(address) } + +// gives a rough estimate of the size this will take up in the db, good enough for maintaining db size limit accurately +func getEstimatedEntrySizeBytes(mizuEntry models.MizuEntry) int { + sizeBytes := len(mizuEntry.Entry) + sizeBytes += len(mizuEntry.EntryId) + sizeBytes += len(mizuEntry.Service) + sizeBytes += len(mizuEntry.Url) + sizeBytes += len(mizuEntry.Method) + sizeBytes += len(mizuEntry.RequestSenderIp) + sizeBytes += len(mizuEntry.ResolvedDestination) + sizeBytes += len(mizuEntry.ResolvedSource) + sizeBytes += 8 // Status bytes (sqlite integer is always 8 bytes) + sizeBytes += 8 // Timestamp bytes + sizeBytes += 8 // SizeBytes bytes + sizeBytes += 1 // IsOutgoing bytes + + + return sizeBytes +} diff --git a/api/pkg/database/main.go b/api/pkg/database/main.go index 4f144400e..921b13442 100644 --- a/api/pkg/database/main.go +++ b/api/pkg/database/main.go @@ -39,6 +39,7 @@ func GetEntriesTable() *gorm.DB { } func initDataBase(databasePath string) *gorm.DB { + go StartEnforcingDatabaseSize( 10 * 1000 * 1000) temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{}) _ = temp.AutoMigrate(&models.MizuEntry{}) // this will ensure table is created return temp diff --git a/api/pkg/database/size_enforcer.go b/api/pkg/database/size_enforcer.go new file mode 100644 index 000000000..f84d461b9 --- /dev/null +++ b/api/pkg/database/size_enforcer.go @@ -0,0 +1,94 @@ +package database + +import ( + "fmt" + "github.com/fsnotify/fsnotify" + "github.com/up9inc/mizu/shared/debounce" + "log" + "mizuserver/pkg/models" + "os" + "time" +) + +const percentageOfMaxSizeBytesToPrune = 5 + +func StartEnforcingDatabaseSize(maxSizeBytes int64) { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal("Error creating filesystem watcher for db size enforcement:", err) + } + defer watcher.Close() + + checkFileSizeDebouncer := debounce.NewDebouncer(5 * time.Second, func() { + checkFileSize(maxSizeBytes) + }) + + done := make(chan bool) + go func() { + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return // closed channel + } + if event.Op&fsnotify.Write == fsnotify.Write { + checkFileSizeDebouncer.SetOn() + } + case err, ok := <-watcher.Errors: + if !ok { + return // closed channel + } + fmt.Printf("filesystem watcher encountered error:%v\n", err) + } + } + }() + + err = watcher.Add(DBPath) + if err != nil { + log.Fatal(fmt.Sprintf("Error adding %s to filesystem watcher for db size enforcement: %v", DBPath, err)) + } + <-done +} + +func checkFileSize(maxSizeBytes int64) { + fileStat, err := os.Stat(DBPath) + if err != nil { + fmt.Printf("Error checking %s file size: %v\n", DBPath, err) + } else { + if fileStat.Size() > maxSizeBytes { + pruneOldEntries(maxSizeBytes) + } + } +} + +func pruneOldEntries(maxSizeBytes int64) { + amountOfBytesToTrim := maxSizeBytes * (percentageOfMaxSizeBytesToPrune / 100) + + rows, err := GetEntriesTable().Limit(100).Order("id").Rows() + if err != nil { + fmt.Printf("Error getting 100 first db rows: %v\n", err) + return + } + + entryIdsToRemove := make([]uint, 0) + bytesToBeRemoved := int64(0) + for rows.Next() { + if bytesToBeRemoved >= amountOfBytesToTrim { + break + } + var entry models.MizuEntry + err = DB.ScanRows(rows, &entry) + if err != nil { + fmt.Printf("Error scanning db row: %v\n", err) + continue + } + + entryIdsToRemove = append(entryIdsToRemove, entry.ID) + bytesToBeRemoved += int64(entry.EstimatedSizeBytes) + } + + if len(entryIdsToRemove) > 0 { + GetEntriesTable().Delete(entryIdsToRemove) + fmt.Printf("Removed %d rows and cleared %d bytes", len(entryIdsToRemove), bytesToBeRemoved) + } +} diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index 8c0d07e22..bf2d5b744 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -33,6 +33,7 @@ type MizuEntry struct { ResolvedSource string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"` ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` IsOutgoing bool `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"` + EstimatedSizeBytes int `json:"-" gorm:"column:estimatedSizeBytes"` } type BaseEntryDetails struct { diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index d3a6a69f8..320b37987 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -3,10 +3,10 @@ package cmd import ( "context" "fmt" - "github.com/up9inc/mizu/cli/debounce" "github.com/up9inc/mizu/cli/kubernetes" "github.com/up9inc/mizu/cli/mizu" "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/shared/debounce" core "k8s.io/api/core/v1" "log" "net/http" diff --git a/cli/debounce/debounce.go b/shared/debounce/debounce.go similarity index 100% rename from cli/debounce/debounce.go rename to shared/debounce/debounce.go From 96f47116f0335eed04d88e4771f713647774b0af Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Tue, 13 Jul 2021 16:21:32 +0300 Subject: [PATCH 02/14] Update go.sum, main.go, and 10 more files... --- api/go.sum | 2 + api/main.go | 21 ++++++++ api/pkg/database/main.go | 15 ++++-- api/pkg/database/size_enforcer.go | 52 ++++++++++++++----- .../messageSensitiveDataCleaner.go | 4 +- cli/cmd/tap.go | 16 ++++++ cli/cmd/tapRunner.go | 24 ++++----- cli/kubernetes/provider.go | 7 ++- shared/consts.go | 1 + shared/go.mod | 6 ++- shared/human_data_sizes.go | 11 ++++ shared/models.go | 1 + 12 files changed, 126 insertions(+), 34 deletions(-) create mode 100644 shared/human_data_sizes.go diff --git a/api/go.sum b/api/go.sum index baf901d28..28981733e 100644 --- a/api/go.sum +++ b/api/go.sum @@ -61,6 +61,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg= github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= diff --git a/api/main.go b/api/main.go index 0a6dc516a..2239b1e75 100644 --- a/api/main.go +++ b/api/main.go @@ -17,6 +17,7 @@ import ( "mizuserver/pkg/utils" "os" "os/signal" + "strings" ) var shouldTap = flag.Bool("tap", false, "Run in tapper mode without API") @@ -126,11 +127,17 @@ func getTrafficFilteringOptions() *shared.TrafficFilteringOptions { return &filteringOptions } +var userAgentsToFilter = []string{"kube-probe", "prometheus"} + func filterHarItems(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) { for message := range inChannel { if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) { continue } + // TODO: move this to tappers + if filterOptions.HideHealthChecks && isHealthCheckByUserAgent(message) { + continue + } sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions) @@ -138,6 +145,20 @@ func filterHarItems(inChannel <- chan *tap.OutputChannelItem, outChannel chan *t } } +func isHealthCheckByUserAgent(message *tap.OutputChannelItem) bool { + for _, header := range message.HarEntry.Request.Headers { + if strings.ToLower(header.Name) == "user-agent" { + for _, userAgent := range userAgentsToFilter { + if strings.Contains(strings.ToLower(header.Value), userAgent) { + return true + } + return false + } + } + } + return false +} + func pipeChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) { if connection == nil { panic("Websocket connection is nil") diff --git a/api/pkg/database/main.go b/api/pkg/database/main.go index 921b13442..58d215bb3 100644 --- a/api/pkg/database/main.go +++ b/api/pkg/database/main.go @@ -4,6 +4,7 @@ import ( "fmt" "gorm.io/driver/sqlite" "gorm.io/gorm" + "gorm.io/gorm/logger" "mizuserver/pkg/models" "mizuserver/pkg/utils" ) @@ -12,9 +13,7 @@ const ( DBPath = "./entries.db" ) -var ( - DB = initDataBase(DBPath) -) + var DB *gorm.DB const ( OrderDesc = "desc" @@ -34,13 +33,19 @@ var ( } ) +func init() { + DB = initDataBase(DBPath) + go StartEnforcingDatabaseSize() +} + func GetEntriesTable() *gorm.DB { return DB.Table("mizu_entries") } func initDataBase(databasePath string) *gorm.DB { - go StartEnforcingDatabaseSize( 10 * 1000 * 1000) - temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{}) + temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) _ = temp.AutoMigrate(&models.MizuEntry{}) // this will ensure table is created return temp } diff --git a/api/pkg/database/size_enforcer.go b/api/pkg/database/size_enforcer.go index f84d461b9..e4841e014 100644 --- a/api/pkg/database/size_enforcer.go +++ b/api/pkg/database/size_enforcer.go @@ -3,24 +3,34 @@ package database import ( "fmt" "github.com/fsnotify/fsnotify" + "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/debounce" "log" "mizuserver/pkg/models" "os" + "strconv" "time" ) -const percentageOfMaxSizeBytesToPrune = 5 +const percentageOfMaxSizeBytesToPrune = 15 +const defaultMaxDatabaseSizeBytes = 200 * 1000 * 1000 -func StartEnforcingDatabaseSize(maxSizeBytes int64) { +func StartEnforcingDatabaseSize() { watcher, err := fsnotify.NewWatcher() if err != nil { - log.Fatal("Error creating filesystem watcher for db size enforcement:", err) + log.Printf("Error creating filesystem watcher for db size enforcement: %v\n", err) // TODO: make fatal + return } defer watcher.Close() - checkFileSizeDebouncer := debounce.NewDebouncer(5 * time.Second, func() { - checkFileSize(maxSizeBytes) + maxEntriesDBByteSize, err := getMaxEntriesDBByteSize() + if err != nil { + log.Printf("Error parsing max db size: %v\n", err) // TODO: make fatal + return + } + + checkFileSizeDebouncer := debounce.NewDebouncer(5*time.Second, func() { + checkFileSize(maxEntriesDBByteSize) }) done := make(chan bool) @@ -45,28 +55,40 @@ func StartEnforcingDatabaseSize(maxSizeBytes int64) { err = watcher.Add(DBPath) if err != nil { - log.Fatal(fmt.Sprintf("Error adding %s to filesystem watcher for db size enforcement: %v", DBPath, err)) + log.Printf("Error adding %s to filesystem watcher for db size enforcement: %v\n", DBPath, err) //TODO: make fatal } <-done } +func getMaxEntriesDBByteSize() (int64, error) { + maxEntriesDBByteSize := int64(defaultMaxDatabaseSizeBytes) + var err error + + maxEntriesDBSizeByteSEnvVarValue := os.Getenv(shared.MaxEntriesDBSizeByteSEnvVar) + if maxEntriesDBSizeByteSEnvVarValue != "" { + maxEntriesDBByteSize, err = strconv.ParseInt(maxEntriesDBSizeByteSEnvVarValue, 10, 64) + } + return maxEntriesDBByteSize, err +} + func checkFileSize(maxSizeBytes int64) { fileStat, err := os.Stat(DBPath) if err != nil { fmt.Printf("Error checking %s file size: %v\n", DBPath, err) } else { + fmt.Printf("%s size is %s, checking if over %s\n", DBPath, shared.BytesToHumanReadable(fileStat.Size()), shared.BytesToHumanReadable(maxSizeBytes)) if fileStat.Size() > maxSizeBytes { - pruneOldEntries(maxSizeBytes) + pruneOldEntries(fileStat.Size()) } } } -func pruneOldEntries(maxSizeBytes int64) { - amountOfBytesToTrim := maxSizeBytes * (percentageOfMaxSizeBytesToPrune / 100) +func pruneOldEntries(currentFileSize int64) { + amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune) - rows, err := GetEntriesTable().Limit(100).Order("id").Rows() + rows, err := GetEntriesTable().Limit(10000).Order("id").Rows() if err != nil { - fmt.Printf("Error getting 100 first db rows: %v\n", err) + fmt.Printf("Error getting 10000 first db rows: %v\n", err) return } @@ -88,7 +110,11 @@ func pruneOldEntries(maxSizeBytes int64) { } if len(entryIdsToRemove) > 0 { - GetEntriesTable().Delete(entryIdsToRemove) - fmt.Printf("Removed %d rows and cleared %d bytes", len(entryIdsToRemove), bytesToBeRemoved) + GetEntriesTable().Where(entryIdsToRemove).Delete(models.MizuEntry{}) + // VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this + DB.Exec("VACUUM") + fmt.Printf("Removed %d rows and cleared %s bytes", len(entryIdsToRemove), shared.BytesToHumanReadable(bytesToBeRemoved)) + } else { + fmt.Printf("Found no rows to remove when pruning") } } diff --git a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go index cc0e4d289..0d9882fb0 100644 --- a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go +++ b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go @@ -15,8 +15,8 @@ import ( ) func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, options *shared.TrafficFilteringOptions) { - harOutputItem.HarEntry.Request.Headers = filterHarHeaders(harOutputItem.HarEntry.Request.Headers) - harOutputItem.HarEntry.Response.Headers = filterHarHeaders(harOutputItem.HarEntry.Response.Headers) + //harOutputItem.HarEntry.Request.Headers = filterHarHeaders(harOutputItem.HarEntry.Request.Headers) + //harOutputItem.HarEntry.Response.Headers = filterHarHeaders(harOutputItem.HarEntry.Response.Headers) harOutputItem.HarEntry.Request.Cookies = make([]har.Cookie, 0, 0) harOutputItem.HarEntry.Response.Cookies = make([]har.Cookie, 0, 0) diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 47f209738..30611bee0 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -3,6 +3,7 @@ package cmd import ( "errors" "fmt" + "github.com/up9inc/mizu/shared" "regexp" "strings" @@ -21,10 +22,15 @@ type MizuTapOptions struct { MizuImage string PlainTextFilterRegexes []string TapOutgoing bool + HideHealthChecks bool + MaxEntriesDBSizeBytes int64 } var mizuTapOptions = &MizuTapOptions{} var direction string +var humanMaxEntriesDBSize string + +const maxEntriesDBSizeFlagName = "max-entries-db-size" var tapCmd = &cobra.Command{ Use: "tap [POD REGEX]", @@ -43,6 +49,14 @@ Supported protocols are HTTP and gRPC.`, return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], err)) } + mizuTapOptions.MaxEntriesDBSizeBytes, err = shared.HumanReadableToBytes(humanMaxEntriesDBSize) + if err != nil { + return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize)) + } else if cmd.Flags().Changed(maxEntriesDBSizeFlagName) { + // We're parsing human readable file sizes here so its best to be unambiguous + fmt.Printf("Setting max entries db size to %s\n", shared.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes)) + } + directionLowerCase := strings.ToLower(direction) if directionLowerCase == "any" { mizuTapOptions.TapOutgoing = true @@ -69,4 +83,6 @@ func init() { tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector") tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "regex-masking", "r", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies") tapCmd.Flags().StringVarP(&direction, "direction", "", "in", "Record traffic that goes in this direction (relative to the tapped pod): in/any") + tapCmd.Flags().BoolVar(&mizuTapOptions.HideHealthChecks, "hide-healthchecks", false, "hides requests with kube-probe or prometheus user-agent headers") + tapCmd.Flags().StringVarP(&humanMaxEntriesDBSize, maxEntriesDBSizeFlagName, "", "200MB", "override the default max entries db size of 200mb") } diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 320b37987..eb63a39e2 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -95,7 +95,7 @@ func createMizuAggregator(ctx context.Context, kubernetesProvider *kubernetes.Pr var err error mizuServiceAccountExists = createRBACIfNecessary(ctx, kubernetesProvider) - _, err = kubernetesProvider.CreateMizuAggregatorPod(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.MizuImage, mizuServiceAccountExists, mizuApiFilteringOptions) + _, err = kubernetesProvider.CreateMizuAggregatorPod(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.MizuImage, mizuServiceAccountExists, mizuApiFilteringOptions, tappingOptions.MaxEntriesDBSizeBytes) if err != nil { fmt.Printf("Error creating mizu collector pod: %v\n", err) return err @@ -111,21 +111,21 @@ func createMizuAggregator(ctx context.Context, kubernetesProvider *kubernetes.Pr } func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.TrafficFilteringOptions, error) { - if tappingOptions.PlainTextFilterRegexes == nil || len(tappingOptions.PlainTextFilterRegexes) == 0 { - return nil, nil - } + var compiledRegexSlice []*shared.SerializableRegexp - compiledRegexSlice := make([]*shared.SerializableRegexp, 0) - for _, regexStr := range tappingOptions.PlainTextFilterRegexes { - compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr) - if err != nil { - fmt.Printf("Regex %s is invalid: %v", regexStr, err) - return nil, err + if tappingOptions.PlainTextFilterRegexes != nil && len(tappingOptions.PlainTextFilterRegexes) > 0 { + compiledRegexSlice = make([]*shared.SerializableRegexp, 0) + for _, regexStr := range tappingOptions.PlainTextFilterRegexes { + compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr) + if err != nil { + fmt.Printf("Regex %s is invalid: %v", regexStr, err) + return nil, err + } + compiledRegexSlice = append(compiledRegexSlice, compiledRegex) } - compiledRegexSlice = append(compiledRegexSlice, compiledRegex) } - return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice}, nil + return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: tappingOptions.HideHealthChecks}, nil } func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error { diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index 0bf40c1b8..ab92dfc17 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -8,6 +8,7 @@ import ( "fmt" "path/filepath" "regexp" + "strconv" "github.com/up9inc/mizu/shared" core "k8s.io/api/core/v1" @@ -70,7 +71,7 @@ func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) w return watcher } -func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace string, podName string, podImage string, linkServiceAccount bool, mizuApiFilteringOptions *shared.TrafficFilteringOptions) (*core.Pod, error) { +func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace string, podName string, podImage string, linkServiceAccount bool, mizuApiFilteringOptions *shared.TrafficFilteringOptions, maxEntriesDBSizeBytes int64) (*core.Pod, error) { marshaledFilteringOptions, err := json.Marshal(mizuApiFilteringOptions) if err != nil { return nil, err @@ -97,6 +98,10 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace Name: shared.MizuFilteringOptionsEnvVar, Value: string(marshaledFilteringOptions), }, + { + Name: shared.MaxEntriesDBSizeByteSEnvVar, + Value: strconv.FormatInt(maxEntriesDBSizeBytes, 10), + }, }, }, }, diff --git a/shared/consts.go b/shared/consts.go index ccda67615..b752b6ed3 100644 --- a/shared/consts.go +++ b/shared/consts.go @@ -5,4 +5,5 @@ const ( HostModeEnvVar = "HOST_MODE" NodeNameEnvVar = "NODE_NAME" TappedAddressesPerNodeDictEnvVar = "TAPPED_ADDRESSES_PER_HOST" + MaxEntriesDBSizeByteSEnvVar = "MAX_ENTRIES_DB_BYTES" ) diff --git a/shared/go.mod b/shared/go.mod index d1afa338c..625391625 100644 --- a/shared/go.mod +++ b/shared/go.mod @@ -2,4 +2,8 @@ module github.com/up9inc/mizu/shared go 1.16 -require github.com/gorilla/websocket v1.4.2 +require ( + github.com/gorilla/websocket v1.4.2 + github.com/docker/go-units v0.4.0 +) + diff --git a/shared/human_data_sizes.go b/shared/human_data_sizes.go new file mode 100644 index 000000000..30596b9fb --- /dev/null +++ b/shared/human_data_sizes.go @@ -0,0 +1,11 @@ +package shared + +import "github.com/docker/go-units" + +func BytesToHumanReadable(bytes int64) string { + return units.HumanSize(float64(bytes)) +} + +func HumanReadableToBytes(humanReadableSize string) (int64, error) { + return units.FromHumanSize(humanReadableSize) +} diff --git a/shared/models.go b/shared/models.go index f1d443cdf..edeb3b51b 100644 --- a/shared/models.go +++ b/shared/models.go @@ -58,4 +58,5 @@ func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSoc type TrafficFilteringOptions struct { PlainTextMaskingRegexes []*SerializableRegexp + HideHealthChecks bool } From 7cd6d123d16b064bcd1658c9f3d6fe0fe3b1b15c Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Tue, 13 Jul 2021 16:29:07 +0300 Subject: [PATCH 03/14] Update size_enforcer.go --- api/pkg/database/size_enforcer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/api/pkg/database/size_enforcer.go b/api/pkg/database/size_enforcer.go index e4841e014..fb2acef96 100644 --- a/api/pkg/database/size_enforcer.go +++ b/api/pkg/database/size_enforcer.go @@ -76,7 +76,6 @@ func checkFileSize(maxSizeBytes int64) { if err != nil { fmt.Printf("Error checking %s file size: %v\n", DBPath, err) } else { - fmt.Printf("%s size is %s, checking if over %s\n", DBPath, shared.BytesToHumanReadable(fileStat.Size()), shared.BytesToHumanReadable(maxSizeBytes)) if fileStat.Size() > maxSizeBytes { pruneOldEntries(fileStat.Size()) } From 400774555aa601216525504e8af2467f5d82394e Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Tue, 13 Jul 2021 16:38:34 +0300 Subject: [PATCH 04/14] Update size_enforcer.go --- api/pkg/database/size_enforcer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/pkg/database/size_enforcer.go b/api/pkg/database/size_enforcer.go index fb2acef96..cfaa293da 100644 --- a/api/pkg/database/size_enforcer.go +++ b/api/pkg/database/size_enforcer.go @@ -112,8 +112,8 @@ func pruneOldEntries(currentFileSize int64) { GetEntriesTable().Where(entryIdsToRemove).Delete(models.MizuEntry{}) // VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this DB.Exec("VACUUM") - fmt.Printf("Removed %d rows and cleared %s bytes", len(entryIdsToRemove), shared.BytesToHumanReadable(bytesToBeRemoved)) + fmt.Printf("Removed %d rows and cleared %s bytes\n", len(entryIdsToRemove), shared.BytesToHumanReadable(bytesToBeRemoved)) } else { - fmt.Printf("Found no rows to remove when pruning") + fmt.Println("Found no rows to remove when pruning") } } From 5b439d831685874903923ab8bf422d5287e9c672 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Tue, 13 Jul 2021 16:42:10 +0300 Subject: [PATCH 05/14] Update size_enforcer.go --- api/pkg/database/size_enforcer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/pkg/database/size_enforcer.go b/api/pkg/database/size_enforcer.go index cfaa293da..3ef291663 100644 --- a/api/pkg/database/size_enforcer.go +++ b/api/pkg/database/size_enforcer.go @@ -76,6 +76,7 @@ func checkFileSize(maxSizeBytes int64) { if err != nil { fmt.Printf("Error checking %s file size: %v\n", DBPath, err) } else { + fmt.Printf("Checking if %s is > %s", shared.BytesToHumanReadable(fileStat.Size()), shared.BytesToHumanReadable(maxSizeBytes)) if fileStat.Size() > maxSizeBytes { pruneOldEntries(fileStat.Size()) } From 4a053734d90f4523398d987c5137aeae2184772b Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Tue, 13 Jul 2021 16:42:19 +0300 Subject: [PATCH 06/14] Update messageSensitiveDataCleaner.go --- api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go index 0d9882fb0..cc0e4d289 100644 --- a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go +++ b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go @@ -15,8 +15,8 @@ import ( ) func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, options *shared.TrafficFilteringOptions) { - //harOutputItem.HarEntry.Request.Headers = filterHarHeaders(harOutputItem.HarEntry.Request.Headers) - //harOutputItem.HarEntry.Response.Headers = filterHarHeaders(harOutputItem.HarEntry.Response.Headers) + harOutputItem.HarEntry.Request.Headers = filterHarHeaders(harOutputItem.HarEntry.Request.Headers) + harOutputItem.HarEntry.Response.Headers = filterHarHeaders(harOutputItem.HarEntry.Response.Headers) harOutputItem.HarEntry.Request.Cookies = make([]har.Cookie, 0, 0) harOutputItem.HarEntry.Response.Cookies = make([]har.Cookie, 0, 0) From bd71e9a1223720f66c30e2fe6bd3f3b3b088a68a Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Tue, 13 Jul 2021 16:43:26 +0300 Subject: [PATCH 07/14] Update size_enforcer.go --- api/pkg/database/size_enforcer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/api/pkg/database/size_enforcer.go b/api/pkg/database/size_enforcer.go index 3ef291663..cfaa293da 100644 --- a/api/pkg/database/size_enforcer.go +++ b/api/pkg/database/size_enforcer.go @@ -76,7 +76,6 @@ func checkFileSize(maxSizeBytes int64) { if err != nil { fmt.Printf("Error checking %s file size: %v\n", DBPath, err) } else { - fmt.Printf("Checking if %s is > %s", shared.BytesToHumanReadable(fileStat.Size()), shared.BytesToHumanReadable(maxSizeBytes)) if fileStat.Size() > maxSizeBytes { pruneOldEntries(fileStat.Size()) } From c4048e5c8e7009776aea204bd0f0d911355db217 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Tue, 13 Jul 2021 16:56:02 +0300 Subject: [PATCH 08/14] Update main.go --- api/main.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/api/main.go b/api/main.go index 13f212627..b9908588a 100644 --- a/api/main.go +++ b/api/main.go @@ -25,13 +25,12 @@ var aggregator = flag.Bool("aggregator", false, "Run in aggregator mode with API var standalone = flag.Bool("standalone", false, "Run in standalone tapper and API mode") var aggregatorAddress = flag.String("aggregator-address", "", "Address of mizu collector for tapping") - func main() { flag.Parse() hostMode := os.Getenv(shared.HostModeEnvVar) == "1" tapOpts := &tap.TapOpts{HostMode: hostMode} - if !*shouldTap && !*aggregator && !*standalone{ + if !*shouldTap && !*aggregator && !*standalone { panic("One of the flags --tap, --api or --standalone must be provided") } @@ -84,7 +83,6 @@ func main() { func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) { app := fiber.New() - middleware.FiberMiddleware(app) // Register Fiber's middleware for app. app.Static("/", "./site") @@ -102,7 +100,6 @@ func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) { utils.StartServer(app) } - func getTapTargets() []string { nodeName := os.Getenv(shared.NodeNameEnvVar) var tappedAddressesPerNodeDict map[string][]string @@ -129,7 +126,7 @@ func getTrafficFilteringOptions() *shared.TrafficFilteringOptions { var userAgentsToFilter = []string{"kube-probe", "prometheus"} -func filterHarItems(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) { +func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) { for message := range inChannel { if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) { continue From 728b5b5d1c5893cd2d12babac8796440205cc856 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Tue, 13 Jul 2021 16:56:39 +0300 Subject: [PATCH 09/14] Update consts.go, go.mod, and 2 more files... --- shared/consts.go | 8 ++++---- shared/go.mod | 4 ++-- shared/go.sum | 2 ++ shared/socket_client.go | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/shared/consts.go b/shared/consts.go index b752b6ed3..0d452910d 100644 --- a/shared/consts.go +++ b/shared/consts.go @@ -1,9 +1,9 @@ package shared const ( - MizuFilteringOptionsEnvVar = "SENSITIVE_DATA_FILTERING_OPTIONS" - HostModeEnvVar = "HOST_MODE" - NodeNameEnvVar = "NODE_NAME" + MizuFilteringOptionsEnvVar = "SENSITIVE_DATA_FILTERING_OPTIONS" + HostModeEnvVar = "HOST_MODE" + NodeNameEnvVar = "NODE_NAME" TappedAddressesPerNodeDictEnvVar = "TAPPED_ADDRESSES_PER_HOST" - MaxEntriesDBSizeByteSEnvVar = "MAX_ENTRIES_DB_BYTES" + MaxEntriesDBSizeByteSEnvVar = "MAX_ENTRIES_DB_BYTES" ) diff --git a/shared/go.mod b/shared/go.mod index 625391625..66e5165d6 100644 --- a/shared/go.mod +++ b/shared/go.mod @@ -3,7 +3,7 @@ module github.com/up9inc/mizu/shared go 1.16 require ( - github.com/gorilla/websocket v1.4.2 - github.com/docker/go-units v0.4.0 + github.com/gorilla/websocket v1.4.2 + github.com/docker/go-units v0.4.0 ) diff --git a/shared/go.sum b/shared/go.sum index 85efffd99..b46c3a514 100644 --- a/shared/go.sum +++ b/shared/go.sum @@ -1,2 +1,4 @@ +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/shared/socket_client.go b/shared/socket_client.go index 2faeabf5c..4cc618356 100644 --- a/shared/socket_client.go +++ b/shared/socket_client.go @@ -7,7 +7,7 @@ import ( ) const ( - DEFAULT_SOCKET_RETRIES = 3 + DEFAULT_SOCKET_RETRIES = 3 DEFAULT_SOCKET_RETRY_SLEEP_TIME = time.Second * 10 ) From 8400e9e9030d5b6d6a933fe63ce576fa128295cb Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Wed, 14 Jul 2021 14:39:09 +0300 Subject: [PATCH 10/14] Update main.go --- api/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/main.go b/api/main.go index b9908588a..342b2d166 100644 --- a/api/main.go +++ b/api/main.go @@ -149,8 +149,8 @@ func isHealthCheckByUserAgent(message *tap.OutputChannelItem) bool { if strings.Contains(strings.ToLower(header.Value), userAgent) { return true } - return false } + return false } } return false From 8886590ea2e6d9dd3ad8e4aa4c81d5375d95281b Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Wed, 14 Jul 2021 17:32:55 +0300 Subject: [PATCH 11/14] Update main.go, main.go, and 5 more files... --- api/main.go | 2 +- api/pkg/database/main.go | 3 +- api/pkg/database/size_enforcer.go | 18 ++++---- api/pkg/utils/truncating_logger.go | 58 ++++++++++++++++++++++++++ cli/cmd/tap.go | 7 ++-- shared/{ => units}/human_data_sizes.go | 2 +- 6 files changed, 74 insertions(+), 16 deletions(-) create mode 100644 api/pkg/utils/truncating_logger.go rename shared/{ => units}/human_data_sizes.go (94%) diff --git a/api/main.go b/api/main.go index 342b2d166..6ca985759 100644 --- a/api/main.go +++ b/api/main.go @@ -131,7 +131,7 @@ func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *ta if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) { continue } - // TODO: move this to tappers + // TODO: move this to tappers https://up9.atlassian.net/browse/TRA-3441 if filterOptions.HideHealthChecks && isHealthCheckByUserAgent(message) { continue } diff --git a/api/pkg/database/main.go b/api/pkg/database/main.go index 58d215bb3..ea4b420f9 100644 --- a/api/pkg/database/main.go +++ b/api/pkg/database/main.go @@ -7,6 +7,7 @@ import ( "gorm.io/gorm/logger" "mizuserver/pkg/models" "mizuserver/pkg/utils" + "time" ) const ( @@ -44,7 +45,7 @@ func GetEntriesTable() *gorm.DB { func initDataBase(databasePath string) *gorm.DB { temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{ - Logger: logger.Default.LogMode(logger.Silent), + Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond}, }) _ = temp.AutoMigrate(&models.MizuEntry{}) // this will ensure table is created return temp diff --git a/api/pkg/database/size_enforcer.go b/api/pkg/database/size_enforcer.go index cfaa293da..f26b1a4db 100644 --- a/api/pkg/database/size_enforcer.go +++ b/api/pkg/database/size_enforcer.go @@ -5,6 +5,7 @@ import ( "github.com/fsnotify/fsnotify" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/debounce" + "github.com/up9inc/mizu/shared/units" "log" "mizuserver/pkg/models" "os" @@ -13,19 +14,18 @@ import ( ) const percentageOfMaxSizeBytesToPrune = 15 -const defaultMaxDatabaseSizeBytes = 200 * 1000 * 1000 +const defaultMaxDatabaseSizeBytes int64 = 200 * 1000 * 1000 func StartEnforcingDatabaseSize() { watcher, err := fsnotify.NewWatcher() if err != nil { - log.Printf("Error creating filesystem watcher for db size enforcement: %v\n", err) // TODO: make fatal + log.Fatalf("Error creating filesystem watcher for db size enforcement: %v\n", err) return } - defer watcher.Close() maxEntriesDBByteSize, err := getMaxEntriesDBByteSize() if err != nil { - log.Printf("Error parsing max db size: %v\n", err) // TODO: make fatal + log.Fatalf("Error parsing max db size: %v\n", err) return } @@ -33,7 +33,6 @@ func StartEnforcingDatabaseSize() { checkFileSize(maxEntriesDBByteSize) }) - done := make(chan bool) go func() { for { select { @@ -41,7 +40,7 @@ func StartEnforcingDatabaseSize() { if !ok { return // closed channel } - if event.Op&fsnotify.Write == fsnotify.Write { + if event.Op == fsnotify.Write { checkFileSizeDebouncer.SetOn() } case err, ok := <-watcher.Errors: @@ -55,13 +54,12 @@ func StartEnforcingDatabaseSize() { err = watcher.Add(DBPath) if err != nil { - log.Printf("Error adding %s to filesystem watcher for db size enforcement: %v\n", DBPath, err) //TODO: make fatal + log.Fatalf("Error adding %s to filesystem watcher for db size enforcement: %v\n", DBPath, err) } - <-done } func getMaxEntriesDBByteSize() (int64, error) { - maxEntriesDBByteSize := int64(defaultMaxDatabaseSizeBytes) + maxEntriesDBByteSize := defaultMaxDatabaseSizeBytes var err error maxEntriesDBSizeByteSEnvVarValue := os.Getenv(shared.MaxEntriesDBSizeByteSEnvVar) @@ -112,7 +110,7 @@ func pruneOldEntries(currentFileSize int64) { GetEntriesTable().Where(entryIdsToRemove).Delete(models.MizuEntry{}) // VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this DB.Exec("VACUUM") - fmt.Printf("Removed %d rows and cleared %s bytes\n", len(entryIdsToRemove), shared.BytesToHumanReadable(bytesToBeRemoved)) + fmt.Printf("Removed %d rows and cleared %s\n", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved)) } else { fmt.Println("Found no rows to remove when pruning") } diff --git a/api/pkg/utils/truncating_logger.go b/api/pkg/utils/truncating_logger.go new file mode 100644 index 000000000..4b6310ebd --- /dev/null +++ b/api/pkg/utils/truncating_logger.go @@ -0,0 +1,58 @@ +package utils + +import ( + "context" + "fmt" + "gorm.io/gorm/logger" + "gorm.io/gorm/utils" + "time" +) + +// TruncatingLogger implements the gorm logger.Interface interface. Its purpose is to act as gorm's logger while truncating logs to a max of 50 characters to minimise the performance impact +type TruncatingLogger struct { + LogLevel logger.LogLevel + SlowThreshold time.Duration +} + +func (truncatingLogger *TruncatingLogger) LogMode(logLevel logger.LogLevel) logger.Interface { + truncatingLogger.LogLevel = logLevel + return truncatingLogger +} + +func (truncatingLogger *TruncatingLogger) Info(_ context.Context, message string, __ ...interface{}) { + if truncatingLogger.LogLevel < logger.Info { + return + } + fmt.Printf("gorm info: %.50s\n", message) +} + +func (truncatingLogger *TruncatingLogger) Warn(_ context.Context, message string, __ ...interface{}) { + if truncatingLogger.LogLevel < logger.Warn { + return + } + fmt.Printf("gorm warning: %.50s\n", message) +} + +func (truncatingLogger *TruncatingLogger) Error(_ context.Context, message string, __ ...interface{}) { + if truncatingLogger.LogLevel < logger.Error { + return + } + fmt.Printf("gorm error: %.50s\n", message) +} + +func (truncatingLogger *TruncatingLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) { + if truncatingLogger.LogLevel == logger.Silent { + return + } + elapsed := time.Since(begin) + if err != nil { + sql, rows := fc() // copied into every condition as this is a potentially heavy operation best done only when necessary + truncatingLogger.Error(ctx, fmt.Sprintf("Error in %s: %v - elapsed: %fs affected rows: %d, sql: %s", utils.FileWithLineNum(), err, elapsed.Seconds(), rows, sql)) + } else if truncatingLogger.LogLevel >= logger.Warn && elapsed > truncatingLogger.SlowThreshold { + sql, rows := fc() + truncatingLogger.Warn(ctx, fmt.Sprintf("Slow sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql)) + } else if truncatingLogger.LogLevel >= logger.Info { + sql, rows := fc() + truncatingLogger.Info(ctx, fmt.Sprintf("Sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql)) + } +} diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 0350c7b17..9418e9a05 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -3,7 +3,7 @@ package cmd import ( "errors" "fmt" - "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/shared/units" "regexp" "strings" @@ -50,12 +50,13 @@ Supported protocols are HTTP and gRPC.`, return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], err)) } - mizuTapOptions.MaxEntriesDBSizeBytes, err = shared.HumanReadableToBytes(humanMaxEntriesDBSize) + mizuTapOptions.MaxEntriesDBSizeBytes = 200 * 1000 * 1000 + mizuTapOptions.MaxEntriesDBSizeBytes, err = units.HumanReadableToBytes(humanMaxEntriesDBSize) if err != nil { return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize)) } else if cmd.Flags().Changed(maxEntriesDBSizeFlagName) { // We're parsing human readable file sizes here so its best to be unambiguous - fmt.Printf("Setting max entries db size to %s\n", shared.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes)) + fmt.Printf("Setting max entries db size to %s\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes)) } directionLowerCase := strings.ToLower(direction) diff --git a/shared/human_data_sizes.go b/shared/units/human_data_sizes.go similarity index 94% rename from shared/human_data_sizes.go rename to shared/units/human_data_sizes.go index 30596b9fb..d3c39bf7c 100644 --- a/shared/human_data_sizes.go +++ b/shared/units/human_data_sizes.go @@ -1,4 +1,4 @@ -package shared +package units import "github.com/docker/go-units" From 4e7bc05ecf884e97842338218e9e06a977832025 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Wed, 14 Jul 2021 17:46:09 +0300 Subject: [PATCH 12/14] Update truncating_logger.go --- api/pkg/utils/truncating_logger.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/pkg/utils/truncating_logger.go b/api/pkg/utils/truncating_logger.go index 4b6310ebd..f3a0c72a6 100644 --- a/api/pkg/utils/truncating_logger.go +++ b/api/pkg/utils/truncating_logger.go @@ -23,21 +23,21 @@ func (truncatingLogger *TruncatingLogger) Info(_ context.Context, message string if truncatingLogger.LogLevel < logger.Info { return } - fmt.Printf("gorm info: %.50s\n", message) + fmt.Printf("gorm info: %.150s\n", message) } func (truncatingLogger *TruncatingLogger) Warn(_ context.Context, message string, __ ...interface{}) { if truncatingLogger.LogLevel < logger.Warn { return } - fmt.Printf("gorm warning: %.50s\n", message) + fmt.Printf("gorm warning: %.150s\n", message) } func (truncatingLogger *TruncatingLogger) Error(_ context.Context, message string, __ ...interface{}) { if truncatingLogger.LogLevel < logger.Error { return } - fmt.Printf("gorm error: %.50s\n", message) + fmt.Printf("gorm error: %.150s\n", message) } func (truncatingLogger *TruncatingLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) { From 62b17c1822a869d261adf9f67817204b64d2758c Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Thu, 15 Jul 2021 09:08:43 +0300 Subject: [PATCH 13/14] Update tap.go --- cli/cmd/tap.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 9418e9a05..c4c5802d8 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -50,7 +50,6 @@ Supported protocols are HTTP and gRPC.`, return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], err)) } - mizuTapOptions.MaxEntriesDBSizeBytes = 200 * 1000 * 1000 mizuTapOptions.MaxEntriesDBSizeBytes, err = units.HumanReadableToBytes(humanMaxEntriesDBSize) if err != nil { return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize)) From 6e14fa95a137cde1dbb6762215f40d98eff643c4 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Thu, 15 Jul 2021 09:16:01 +0300 Subject: [PATCH 14/14] Update tap.go and go.sum --- cli/cmd/tap.go | 9 +++++---- cli/go.sum | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 18c057b54..46cb4b69e 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -3,13 +3,13 @@ package cmd import ( "errors" "fmt" + "github.com/spf13/cobra" "github.com/up9inc/mizu/cli/mizu" "github.com/up9inc/mizu/cli/uiUtils" + "github.com/up9inc/mizu/shared/units" "os" "regexp" "strings" - "github.com/up9inc/mizu/shared/units" - "github.com/spf13/cobra" ) type MizuTapOptions struct { @@ -60,8 +60,9 @@ Supported protocols are HTTP and gRPC.`, return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], compileErr)) } - mizuTapOptions.MaxEntriesDBSizeBytes, err = units.HumanReadableToBytes(humanMaxEntriesDBSize) - if err != nil { + var parseHumanDataSizeErr error + mizuTapOptions.MaxEntriesDBSizeBytes, parseHumanDataSizeErr = units.HumanReadableToBytes(humanMaxEntriesDBSize) + if parseHumanDataSizeErr != nil { return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize)) } else if cmd.Flags().Changed(maxEntriesDBSizeFlagName) { // We're parsing human readable file sizes here so its best to be unambiguous diff --git a/cli/go.sum b/cli/go.sum index f26ffa68e..9a79ea6a5 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -90,6 +90,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=