Add option to supply user agents to ignore via config (#173)

README.md

@@ -75,7 +75,7 @@ To tap all pods in current namespace -
 
 
 To tap specific pod - 
-``` 
+```bash
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
  front-end-649fc5fd6-kqbtn       2/2     Running   0          7m
@@ -88,7 +88,7 @@ To tap specific pod -
 ```
 
 To tap multiple pods using regex - 
-``` 
+```bash
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
  carts-66c77f5fbb-fq65r          2/2     Running   0          20m
@@ -133,3 +133,17 @@ to the namespace set by `mizu-resources-namespace`. The user must set the tapped
 using the `--namespace` flag or by setting `tap.namespaces` in the config file.
 
 Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior.
+
+### User agent filtering
+
+User-agent filtering (like health checks) - can be configured:
+
+Any request that contains one of those values in the user-agent header will not be captured
+
+```bash
+$ mizu tap "^ca.*" --set ignored-user-agents=kube-probe --set ignored-user-agents=prometheus
++carts-66c77f5fbb-fq65r
++catalogue-5f4cb7cf5-7zrmn
+Web interface is now available at http://localhost:8899
+^C
+```

agent/go.sum

@@ -540,8 +540,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM=
 gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw=
 gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=

agent/main.go

@@ -149,15 +149,13 @@ func getTrafficFilteringOptions() *shared.TrafficFilteringOptions {
 	return &filteringOptions
 }
 
-var userAgentsToFilter = []string{"kube-probe", "prometheus"}
-
 func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) {
 	for message := range inChannel {
 		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
 			continue
 		}
 		// TODO: move this to tappers https://up9.atlassian.net/browse/TRA-3441
-		if filterOptions.HideHealthChecks && isHealthCheckByUserAgent(message) {
+		if isHealthCheckByUserAgent(message, filterOptions.HealthChecksUserAgentHeaders) {
 			continue
 		}
 
@@ -169,11 +167,11 @@ func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *ta
 	}
 }
 
-func isHealthCheckByUserAgent(message *tap.OutputChannelItem) bool {
+func isHealthCheckByUserAgent(message *tap.OutputChannelItem, userAgentsToIgnore []string) bool {
 	for _, header := range message.HarEntry.Request.Headers {
 		if strings.ToLower(header.Name) == "user-agent" {
-			for _, userAgent := range userAgentsToFilter {
+			for _, userAgent := range userAgentsToIgnore {
-				if strings.Contains(strings.ToLower(header.Value), userAgent) {
+				if strings.Contains(strings.ToLower(header.Value), strings.ToLower(userAgent)) {
 					return true
 				}
 			}

cli/cmd/tap.go

@@ -64,7 +64,6 @@ func init() {
 	tapCmd.Flags().Bool(configStructs.AnalysisTapName, defaultTapConfig.Analysis, "Uploads traffic to UP9 for further analysis (Beta)")
 	tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
 	tapCmd.Flags().StringArrayP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
-	tapCmd.Flags().Bool(configStructs.HideHealthChecksTapName, defaultTapConfig.HideHealthChecks, "Hides requests with kube-probe or prometheus user-agent headers")
 	tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
 	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
 	tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any")

cli/cmd/tapRunner.go

@@ -207,7 +207,11 @@ func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) {
 		}
 	}
 
-	return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: mizu.Config.Tap.HideHealthChecks, DisableRedaction: mizu.Config.Tap.DisableRedaction}, nil
+	return &shared.TrafficFilteringOptions{
+		PlainTextMaskingRegexes:      compiledRegexSlice,
+		HealthChecksUserAgentHeaders: mizu.Config.Tap.HealthChecksUserAgentHeaders,
+		DisableRedaction:             mizu.Config.Tap.DisableRedaction,
+	}, nil
 }
 
 func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string) error {

cli/go.sum

@@ -217,7 +217,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -411,10 +410,9 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
-github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=

cli/mizu/config.go

@@ -31,6 +31,7 @@ var allowedSetFlags = []string{
 	KubeConfigPathName,
 	configStructs.AnalysisDestinationTapName,
 	configStructs.SleepIntervalSecTapName,
+	configStructs.IgnoredUserAgentsTapName,
 }
 
 var Config = ConfigStruct{}

cli/mizu/configStructs/tapConfig.go

@@ -17,8 +17,8 @@ const (
 	AnalysisTapName               = "analysis"
 	AllNamespacesTapName          = "all-namespaces"
 	PlainTextFilterRegexesTapName = "regex-masking"
-	HideHealthChecksTapName       = "hide-healthchecks"
 	DisableRedactionTapName       = "no-redact"
+	IgnoredUserAgentsTapName      = "ignored-user-agents"
 	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
 	DirectionTapName              = "direction"
 	DryRunTapName                 = "dry-run"
@@ -26,20 +26,20 @@ const (
 )
 
 type TapConfig struct {
-	AnalysisDestination    string   `yaml:"dest" default:"up9.app"`
+	AnalysisDestination          string   `yaml:"dest" default:"up9.app"`
-	SleepIntervalSec       int      `yaml:"upload-interval" default:"10"`
+	SleepIntervalSec             int      `yaml:"upload-interval" default:"10"`
-	PodRegexStr            string   `yaml:"regex" default:".*"`
+	PodRegexStr                  string   `yaml:"regex" default:".*"`
-	GuiPort                uint16   `yaml:"gui-port" default:"8899"`
+	GuiPort                      uint16   `yaml:"gui-port" default:"8899"`
-	Namespaces             []string `yaml:"namespaces"`
+	Namespaces                   []string `yaml:"namespaces"`
-	Analysis               bool     `yaml:"analysis" default:"false"`
+	Analysis                     bool     `yaml:"analysis" default:"false"`
-	AllNamespaces          bool     `yaml:"all-namespaces" default:"false"`
+	AllNamespaces                bool     `yaml:"all-namespaces" default:"false"`
-	PlainTextFilterRegexes []string `yaml:"regex-masking"`
+	PlainTextFilterRegexes       []string `yaml:"regex-masking"`
-	HideHealthChecks       bool     `yaml:"hide-healthchecks" default:"false"`
+	HealthChecksUserAgentHeaders []string `yaml:"ignored-user-agents" default:"[]"`
-	DisableRedaction       bool     `yaml:"no-redact" default:"false"`
+	DisableRedaction             bool     `yaml:"no-redact" default:"false"`
-	HumanMaxEntriesDBSize  string   `yaml:"max-entries-db-size" default:"200MB"`
+	HumanMaxEntriesDBSize        string   `yaml:"max-entries-db-size" default:"200MB"`
-	Direction              string   `yaml:"direction" default:"in"`
+	Direction                    string   `yaml:"direction" default:"in"`
-	DryRun                 bool     `yaml:"dry-run" default:"false"`
+	DryRun                       bool     `yaml:"dry-run" default:"false"`
-	EnforcePolicyFile      string   `yaml:"test-rules"`
+	EnforcePolicyFile            string   `yaml:"test-rules"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {

shared/go.mod

@@ -3,8 +3,7 @@ module github.com/up9inc/mizu/shared
 go 1.16
 
 require (
-	github.com/google/martian v2.1.0+incompatible // indirect
-	github.com/gorilla/websocket v1.4.2
-	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect
 	github.com/docker/go-units v0.4.0
+	github.com/gorilla/websocket v1.4.2
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

shared/go.sum

@@ -1,8 +1,8 @@
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

shared/models.go

@@ -75,9 +75,9 @@ func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSoc
 }
 
 type TrafficFilteringOptions struct {
-	PlainTextMaskingRegexes []*SerializableRegexp
+	HealthChecksUserAgentHeaders []string
-	HideHealthChecks        bool
+	PlainTextMaskingRegexes      []*SerializableRegexp
-	DisableRedaction        bool
+	DisableRedaction             bool
 }
 
 type VersionResponse struct {