Update issue templates (#189)

Removed hardump flag and made it the default (and only) behavior.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Run mizu <command> '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Logs**
+Upload logs:
+1. Run the mizu command with `--set dump-logs=true` (e.g `mizu tap --set dump-logs=true`) 
+2. Try to reproduce the issue
+3. CNTRL+C on terminal tab which runs mizu
+4. Upload the logs zip file from ~/.mizu/mizu_logs_**.zip
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome]
+
+**Additional context**
+Add any other context about the problem here.

.github/workflows/test.yaml

@@ -0,0 +1,22 @@
+name: test
+on:
+  pull_request:
+    branches:
+      - 'develop'
+      - 'main'
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Test
+        run: make test
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2

Dockerfile

@@ -48,8 +48,6 @@ WORKDIR /app
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
 COPY --from=site-build ["/app/ui-build/build", "site"]
 
-COPY agent/start.sh .
-
 # gin-gonic runs in debug mode without this
 ENV GIN_MODE=release
 

Makefile

@@ -65,3 +65,5 @@ clean-cli:  ## Clean CLI.
 clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
+test: ## Run tests.
+	@echo "running cli tests"; cd cli && $(MAKE) test

PERMISSIONS.md

@@ -0,0 +1,328 @@
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+# Kubernetes permissions for MIZU  
+
+This document describes in details all permissions required for full and correct operation of Mizu
+
+We broke down this list into few categories:
+- Required - what is needed for `mizu` to run properly on your k8s cluster
+- Optional - permissions needed for proper name resolving for service & pod IPs 
+   - addition required for policy validation 
+ 
+
+
+# Required permissions
+
+Mizu needs following permissions on your Kubernetes cluster to run properly
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+```
+
+## Permissions required for service / pod name resolving (opt)
+
+Optionally, for proper resolving of IP addresses to Kubernetes service name, Mizu needs below permissions:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterroles
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterrolebindings
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  - apps
+  - extensions
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+```
+
+## Permissions for Policy rules validation feature (opt)
+
+Optionally, in order to use the policy rules validation feature, Mizu requires the following additional permissions:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - create
+  - delete
+```
+
+- - -
+
+## Namespace-Restricted mode
+
+Alternatively, in order to restrict Mizu to one namespace only (by setting `agent.namespace` in the config file), Mizu needs the following permissions in that namespace:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+```
+
+### Name resolving in Namespace-Restricted mode (opt)
+
+To restrict Mizu to one namespace while also resolving IPs, Mizu needs the following permissions in that namespace:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  - apps
+  - extensions
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+```

README.md

@@ -1,4 +1,5 @@
 ![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+
 # The API Traffic Viewer for Kubernetes
 
 A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined.
@@ -39,317 +40,15 @@ Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page.
 
 ## Prerequisites
 1. Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config`
-2. Mizu needs following permissions on your Kubernetes cluster to run
+2. `mizu` assumes user running the command has permissions to create resources (such as pods, services, namespaces) on your Kubernetes cluster (no worries - `mizu` resources are cleaned up upon termination)
 
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - create
-  - delete
-- apiGroups:
-  - apps
-  resources:
-  - daemonsets
-  verbs:
-  - create
-  - patch
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - namespaces
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services/proxy
-  verbs:
-  - get
-```
+For detailed list of k8s permissions see [PERMISSIONS](PERMISSIONS.md) document
 
-3. Optionally, for resolving traffic IP to Kubernetes service name, Mizu needs below permissions
-
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - apps
-  resources:
-  - daemonsets
-  verbs:
-  - create
-  - patch
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - namespaces
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services/proxy
-  verbs:
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - serviceaccounts
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - clusterroles
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - clusterrolebindings
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - roles
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - rolebindings
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - apps
-  - extensions
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - apps
-  - extensions
-  resources:
-  - services
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  - apps
-  - extensions
-  resources:
-  - endpoints
-  verbs:
-  - get
-  - list
-  - watch
-```
-
-4. Optionally, in order to use the policy rules validation feature, Mizu requires the following additional permissions:
-
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - get
-  - create
-  - delete
-```
-
-5. Alternatively, in order to restrict Mizu to one namespace only (by setting `agent.namespace` in the config file), Mizu needs the following permissions in that namespace:
-
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - apps
-  resources:
-  - daemonsets
-  verbs:
-  - get
-  - create
-  - patch
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services/proxy
-  verbs:
-  - get
-```
-
-6. To restrict Mizu to one namespace while also resolving IPs, Mizu needs the following permissions in that namespace:
-
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - apps
-  resources:
-  - daemonsets
-  verbs:
-  - get
-  - create
-  - patch
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services/proxy
-  verbs:
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - serviceaccounts
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - roles
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - rolebindings
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - apps
-  - extensions
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - apps
-  - extensions
-  resources:
-  - services
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  - apps
-  - extensions
-  resources:
-  - endpoints
-  verbs:
-  - get
-  - list
-  - watch
-```
-
-See `examples/roles` for example `clusterroles`. 
 
 ## How to Run
 
 1. Find pods you'd like to tap to in your Kubernetes cluster
-2. Run `mizu tap PODNAME` or `mizu tap REGEX` 
+2. Run `mizu tap` or `mizu tap PODNAME`  
 3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI .. 
 4. Watch the API traffic flowing ..
 5. Type ^C to stop
@@ -358,9 +57,26 @@ See `examples/roles` for example `clusterroles`.
 
 Run `mizu help` for usage options
 
+To tap all pods in current namespace - 
+``` 
+ $ kubectl get pods 
+ NAME                            READY   STATUS    RESTARTS   AGE
+ carts-66c77f5fbb-fq65r          2/2     Running   0          20m
+ catalogue-5f4cb7cf5-7zrmn       2/2     Running   0          20m
+ front-end-649fc5fd6-kqbtn       2/2     Running   0          20m
+ ..
+
+ $ mizu tap
+ +carts-66c77f5fbb-fq65r
+ +catalogue-5f4cb7cf5-7zrmn
+ +front-end-649fc5fd6-kqbtn
+ Web interface is now available at http://localhost:8899
+ ^C
+```
+
 
 To tap specific pod - 
-``` 
+```bash
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
  front-end-649fc5fd6-kqbtn       2/2     Running   0          7m
@@ -373,7 +89,7 @@ To tap specific pod -
 ```
 
 To tap multiple pods using regex - 
-``` 
+```bash
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
  carts-66c77f5fbb-fq65r          2/2     Running   0          20m
@@ -388,6 +104,22 @@ To tap multiple pods using regex -
  ^C
 ```
 
+## Configuration
+
+Mizu can work with config file which should be stored in ${HOME}/.mizu/config.yaml (macOS: ~/.mizu/config.yaml) <br />
+In case no config file found, defaults will be used. <br />
+In case of partial configuration defined, all other fields will be used with defaults. <br />
+You can always override the defaults or config file with CLI flags.
+
+To get the default config params run `mizu config` <br />
+To generate a new config file with default values use `mizu config -r`
+
+Mizu has several undocumented flags which can be set by using --set flag (e.g., `mizu tap --set dump-logs=true`)
+* **mizu-resources-namespace**: Type - String, See [Namespace-Restricted Mode](#namespace-restricted-mode)
+* **telemetry**: Type - Boolean, Reports telemetry
+* **dump-logs**: Type - Boolean, At the end of the execution it creates a zip file with logs (in .mizu folder)
+* **kube-config-path**: Type - String, Setting the path to kube config (which isn't in standard path)
+
 ## Advanced Usage
 
 ### Namespace-Restricted Mode
@@ -402,3 +134,17 @@ to the namespace set by `mizu-resources-namespace`. The user must set the tapped
 using the `--namespace` flag or by setting `tap.namespaces` in the config file.
 
 Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior.
+
+### User agent filtering
+
+User-agent filtering (like health checks) - can be configured:
+
+Any request that contains one of those values in the user-agent header will not be captured
+
+```bash
+$ mizu tap "^ca.*" --set ignored-user-agents=kube-probe --set ignored-user-agents=prometheus
++carts-66c77f5fbb-fq65r
++catalogue-5f4cb7cf5-7zrmn
+Web interface is now available at http://localhost:8899
+^C
+```

agent/go.sum

@@ -540,8 +540,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM=
 gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw=
 gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=

agent/main.go

@@ -149,15 +149,13 @@ func getTrafficFilteringOptions() *shared.TrafficFilteringOptions {
 	return &filteringOptions
 }
 
-var userAgentsToFilter = []string{"kube-probe", "prometheus"}
-
 func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) {
 	for message := range inChannel {
 		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
 			continue
 		}
 		// TODO: move this to tappers https://up9.atlassian.net/browse/TRA-3441
-		if filterOptions.HideHealthChecks && isHealthCheckByUserAgent(message) {
+		if isHealthCheckByUserAgent(message, filterOptions.HealthChecksUserAgentHeaders) {
 			continue
 		}
 
@@ -169,11 +167,11 @@ func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *ta
 	}
 }
 
-func isHealthCheckByUserAgent(message *tap.OutputChannelItem) bool {
+func isHealthCheckByUserAgent(message *tap.OutputChannelItem, userAgentsToIgnore []string) bool {
 	for _, header := range message.HarEntry.Request.Headers {
 		if strings.ToLower(header.Name) == "user-agent" {
-			for _, userAgent := range userAgentsToFilter {
-				if strings.Contains(strings.ToLower(header.Value), userAgent) {
+			for _, userAgent := range userAgentsToIgnore {
+				if strings.Contains(strings.ToLower(header.Value), strings.ToLower(userAgent)) {
 					return true
 				}
 			}

agent/pkg/controllers/entries_controller.go

@@ -57,7 +57,7 @@ func GetEntries(c *gin.Context) {
 }
 
 func GetHARs(c *gin.Context) {
-	entriesFilter := &models.HarFetchRequestBody{}
+	entriesFilter := &models.HarFetchRequestQuery{}
 	order := database.OrderDesc
 	if err := c.BindQuery(entriesFilter); err != nil {
 		c.JSON(http.StatusBadRequest, err)
@@ -146,12 +146,12 @@ func GetHARs(c *gin.Context) {
 func UploadEntries(c *gin.Context) {
 	rlog.Infof("Upload entries - started\n")
 
-	uploadRequestBody := &models.UploadEntriesRequestBody{}
-	if err := c.BindQuery(uploadRequestBody); err != nil {
+	uploadParams := &models.UploadEntriesRequestQuery{}
+	if err := c.BindQuery(uploadParams); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
-	if err := validation.Validate(uploadRequestBody); err != nil {
+	if err := validation.Validate(uploadParams); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
@@ -160,19 +160,19 @@ func UploadEntries(c *gin.Context) {
 		return
 	}
 
-	rlog.Infof("Upload entries - creating token. dest %s\n", uploadRequestBody.Dest)
-	token, err := up9.CreateAnonymousToken(uploadRequestBody.Dest)
+	rlog.Infof("Upload entries - creating token. dest %s\n", uploadParams.Dest)
+	token, err := up9.CreateAnonymousToken(uploadParams.Dest)
 	if err != nil {
 		c.String(http.StatusServiceUnavailable, "Cannot analyze, mizu is already analyzing")
 		return
 	}
 	rlog.Infof("Upload entries - uploading. token: %s model: %s\n", token.Token, token.Model)
-	go up9.UploadEntriesImpl(token.Token, token.Model, uploadRequestBody.Dest, uploadRequestBody.SleepIntervalSec)
+	go up9.UploadEntriesImpl(token.Token, token.Model, uploadParams.Dest, uploadParams.SleepIntervalSec)
 	c.String(http.StatusOK, "OK")
 }
 
 func GetFullEntries(c *gin.Context) {
-	entriesFilter := &models.HarFetchRequestBody{}
+	entriesFilter := &models.HarFetchRequestQuery{}
 	if err := c.BindQuery(entriesFilter); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 	}

agent/pkg/models/models.go

@@ -119,19 +119,19 @@ func (fedex *FullEntryDetailsExtra) UnmarshalData(entry *MizuEntry) error {
 }
 
 type EntriesFilter struct {
-	Limit     int    `query:"limit" validate:"required,min=1,max=200"`
-	Operator  string `query:"operator" validate:"required,oneof='lt' 'gt'"`
-	Timestamp int64  `query:"timestamp" validate:"required,min=1"`
+	Limit     int    `form:"limit" validate:"required,min=1,max=200"`
+	Operator  string `form:"operator" validate:"required,oneof='lt' 'gt'"`
+	Timestamp int64  `form:"timestamp" validate:"required,min=1"`
 }
 
-type UploadEntriesRequestBody struct {
+type UploadEntriesRequestQuery struct {
 	Dest             string `form:"dest"`
 	SleepIntervalSec int    `form:"interval"`
 }
 
-type HarFetchRequestBody struct {
-	From int64 `query:"from"`
-	To   int64 `query:"to"`
+type HarFetchRequestQuery struct {
+	From int64 `form:"from"`
+	To   int64 `form:"to"`
 }
 
 type WebSocketEntryMessage struct {

agent/start.sh

@@ -1,2 +0,0 @@
-#!/bin/bash
-./mizuagent -i any -hardump -targets ${TAPPED_ADDRESSES}

cli/Makefile

@@ -39,3 +39,6 @@ build-all: ## Build for all supported platforms.
 clean: ## Clean all build artifacts.
 	go clean
 	rm -rf ./bin/*
+
+test: ## Run cli tests.
+	@go test ./... -race -coverprofile=coverage.out -covermode=atomic

cli/README.md

@@ -1,26 +0,0 @@
-# mizu CLI
-## Usage
-`./mizu {pod_name_regex}`
-
-### Optional Flags
-
-| flag                 | default          | purpose                                                                                                      |
-|----------------------|------------------|--------------------------------------------------------------------------------------------------------------|
-| `--no-gui`           | `false`          | Don't host the web interface (not applicable at the moment)                                                      |
-| `--gui-port`         | `8899`           | local port that web interface will be forwarded to                                                               |
-| `--namespace`        |                  | use namespace different than the one found in kubeconfig                                                     |
-| `--kubeconfig`       |                  | Path to custom kubeconfig file                                                                               |
-
-There are some extra flags defined in code that will show up in `./mizu --help`, these are non functional stubs for now
-
-## Installation
-Make sure your go version is at least 1.11
-1. cd to `mizu/cli`
-2. Run `go mod download` (may take a moment)
-3. Run `go build mizu.go`
-
-Alternatively, you can build+run directly using `go run mizu.go {pod_name_regex}`
-
-
-## Known issues
-* mid-flight port forwarding failures are not detected and no indication will be shown when this occurs

cli/cmd/fetch.go

@@ -12,7 +12,7 @@ var fetchCmd = &cobra.Command{
 	Short: "Download recorded traffic to files",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		go mizu.ReportRun("fetch", mizu.Config.Fetch)
-		if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.Fetch.MizuPort); err != nil {
+		if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.Fetch.GuiPort); err != nil {
 			return err
 		} else if !isCompatible {
 			return nil
@@ -31,5 +31,5 @@ func init() {
 	fetchCmd.Flags().StringP(configStructs.DirectoryFetchName, "d", defaultFetchConfig.Directory, "Provide a custom directory for fetched entries")
 	fetchCmd.Flags().Int(configStructs.FromTimestampFetchName, defaultFetchConfig.FromTimestamp, "Custom start timestamp for fetched entries")
 	fetchCmd.Flags().Int(configStructs.ToTimestampFetchName, defaultFetchConfig.ToTimestamp, "Custom end timestamp fetched entries")
-	fetchCmd.Flags().Uint16P(configStructs.MizuPortFetchName, "p", defaultFetchConfig.MizuPort, "Custom port for mizu")
+	fetchCmd.Flags().Uint16P(configStructs.GuiPortFetchName, "p", defaultFetchConfig.GuiPort, "Provide a custom port for the web interface webserver")
 }

cli/cmd/fetchRunner.go

@@ -16,7 +16,7 @@ import (
 )
 
 func RunMizuFetch() {
-	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Fetch.MizuPort)
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Fetch.GuiPort)
 	resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, mizu.Config.Fetch.FromTimestamp, mizu.Config.Fetch.ToTimestamp))
 	if err != nil {
 		log.Fatal(err)

cli/cmd/tap.go

@@ -64,9 +64,8 @@ func init() {
 	tapCmd.Flags().Bool(configStructs.AnalysisTapName, defaultTapConfig.Analysis, "Uploads traffic to UP9 for further analysis (Beta)")
 	tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
 	tapCmd.Flags().StringArrayP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
-	tapCmd.Flags().Bool(configStructs.HideHealthChecksTapName, defaultTapConfig.HideHealthChecks, "hides requests with kube-probe or prometheus user-agent headers")
 	tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
-	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "override the default max entries db size of 200mb")
+	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
 	tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any")
 	tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them")
 	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file with policy rules")

cli/cmd/tapRunner.go

@@ -207,7 +207,11 @@ func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) {
 		}
 	}
 
-	return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: mizu.Config.Tap.HideHealthChecks, DisableRedaction: mizu.Config.Tap.DisableRedaction}, nil
+	return &shared.TrafficFilteringOptions{
+		PlainTextMaskingRegexes:      compiledRegexSlice,
+		HealthChecksUserAgentHeaders: mizu.Config.Tap.HealthChecksUserAgentHeaders,
+		DisableRedaction:             mizu.Config.Tap.DisableRedaction,
+	}, nil
 }
 
 func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string) error {

cli/go.sum

@@ -217,7 +217,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -411,10 +410,9 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
-github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=

cli/kubernetes/provider.go

@@ -577,7 +577,6 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 		"./mizuagent",
 		"-i", "any",
 		"--tap",
-		"--hardump",
 		"--api-server-address", fmt.Sprintf("ws://%s/wsTapper", apiServerPodIp),
 	}
 	if tapOutgoing {

cli/mizu/config.go

@@ -31,6 +31,7 @@ var allowedSetFlags = []string{
 	KubeConfigPathName,
 	configStructs.AnalysisDestinationTapName,
 	configStructs.SleepIntervalSecTapName,
+	configStructs.IgnoredUserAgentsTapName,
 }
 
 var Config = ConfigStruct{}
@@ -69,6 +70,10 @@ func GetConfigWithDefaults() (string, error) {
 	if err := defaults.Set(&defaultConf); err != nil {
 		return "", err
 	}
+
+	// TODO: change to generic solution
+	defaultConf.AgentImage = ""
+
 	return uiUtils.PrettyYaml(defaultConf)
 }
 

cli/mizu/configStruct.go

@@ -19,7 +19,7 @@ type ConfigStruct struct {
 	Fetch                  configStructs.FetchConfig   `yaml:"fetch"`
 	Version                configStructs.VersionConfig `yaml:"version"`
 	View                   configStructs.ViewConfig    `yaml:"view"`
-	AgentImage             string                      `yaml:"agent-image"`
+	AgentImage             string                      `yaml:"agent-image,omitempty"`
 	MizuResourcesNamespace string                      `yaml:"mizu-resources-namespace" default:"mizu"`
 	Telemetry              bool                        `yaml:"telemetry" default:"true"`
 	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`

cli/mizu/configStructs/fetchConfig.go

@@ -4,12 +4,12 @@ const (
 	DirectoryFetchName     = "directory"
 	FromTimestampFetchName = "from"
 	ToTimestampFetchName   = "to"
-	MizuPortFetchName      = "port"
+	GuiPortFetchName       = "gui-port"
 )
 
 type FetchConfig struct {
 	Directory     string `yaml:"directory" default:"."`
 	FromTimestamp int    `yaml:"from" default:"0"`
 	ToTimestamp   int    `yaml:"to" default:"0"`
-	MizuPort      uint16 `yaml:"port" default:"8899"`
+	GuiPort       uint16 `yaml:"gui-port" default:"8899"`
 }

cli/mizu/configStructs/tapConfig.go

@@ -17,8 +17,8 @@ const (
 	AnalysisTapName               = "analysis"
 	AllNamespacesTapName          = "all-namespaces"
 	PlainTextFilterRegexesTapName = "regex-masking"
-	HideHealthChecksTapName       = "hide-healthchecks"
 	DisableRedactionTapName       = "no-redact"
+	IgnoredUserAgentsTapName      = "ignored-user-agents"
 	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
 	DirectionTapName              = "direction"
 	DryRunTapName                 = "dry-run"
@@ -26,20 +26,20 @@ const (
 )
 
 type TapConfig struct {
-	AnalysisDestination    string   `yaml:"dest" default:"up9.app"`
-	SleepIntervalSec       int      `yaml:"upload-interval" default:"10"`
-	PodRegexStr            string   `yaml:"regex" default:".*"`
-	GuiPort                uint16   `yaml:"gui-port" default:"8899"`
-	Namespaces             []string `yaml:"namespaces"`
-	Analysis               bool     `yaml:"analysis" default:"false"`
-	AllNamespaces          bool     `yaml:"all-namespaces" default:"false"`
-	PlainTextFilterRegexes []string `yaml:"regex-masking"`
-	HideHealthChecks       bool     `yaml:"hide-healthchecks" default:"false"`
-	DisableRedaction       bool     `yaml:"no-redact" default:"false"`
-	HumanMaxEntriesDBSize  string   `yaml:"max-entries-db-size" default:"200MB"`
-	Direction              string   `yaml:"direction" default:"in"`
-	DryRun                 bool     `yaml:"dry-run" default:"false"`
-	EnforcePolicyFile      string   `yaml:"test-rules"`
+	AnalysisDestination          string   `yaml:"dest" default:"up9.app"`
+	SleepIntervalSec             int      `yaml:"upload-interval" default:"10"`
+	PodRegexStr                  string   `yaml:"regex" default:".*"`
+	GuiPort                      uint16   `yaml:"gui-port" default:"8899"`
+	Namespaces                   []string `yaml:"namespaces"`
+	Analysis                     bool     `yaml:"analysis" default:"false"`
+	AllNamespaces                bool     `yaml:"all-namespaces" default:"false"`
+	PlainTextFilterRegexes       []string `yaml:"regex-masking"`
+	HealthChecksUserAgentHeaders []string `yaml:"ignored-user-agents" default:"[]"`
+	DisableRedaction             bool     `yaml:"no-redact" default:"false"`
+	HumanMaxEntriesDBSize        string   `yaml:"max-entries-db-size" default:"200MB"`
+	Direction                    string   `yaml:"direction" default:"in"`
+	DryRun                       bool     `yaml:"dry-run" default:"false"`
+	EnforcePolicyFile            string   `yaml:"test-rules"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {

cli/mizu/sliceUtils_test.go

@@ -0,0 +1,82 @@
+package mizu_test
+
+import (
+	"github.com/up9inc/mizu/cli/mizu"
+	"testing"
+)
+
+func TestContainsExists(t *testing.T) {
+	tests := []struct {
+		slice         []string
+		containsValue string
+		expected      bool
+	}{
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "apple", expected: true},
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "orange", expected: true},
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "banana", expected: true},
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "grapes", expected: true},
+	}
+
+	for _, test := range tests {
+		actual := mizu.Contains(test.slice, test.containsValue)
+		if actual != test.expected {
+			t.Errorf("unexpected result - expected: %v, actual: %v", test.expected, actual)
+		}
+	}
+}
+
+func TestContainsNotExists(t *testing.T) {
+	tests := []struct {
+		slice         []string
+		containsValue string
+		expected      bool
+	}{
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "cat", expected: false},
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "dog", expected: false},
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "apples", expected: false},
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "rapes", expected: false},
+	}
+
+	for _, test := range tests {
+		actual := mizu.Contains(test.slice, test.containsValue)
+		if actual != test.expected {
+			t.Errorf("unexpected result - expected: %v, actual: %v", test.expected, actual)
+		}
+	}
+}
+
+func TestContainsEmptySlice(t *testing.T) {
+	tests := []struct {
+		slice         []string
+		containsValue string
+		expected      bool
+	}{
+		{slice: []string{}, containsValue: "cat", expected: false},
+		{slice: []string{}, containsValue: "dog", expected: false},
+	}
+
+	for _, test := range tests {
+		actual := mizu.Contains(test.slice, test.containsValue)
+		if actual != test.expected {
+			t.Errorf("unexpected result - expected: %v, actual: %v", test.expected, actual)
+		}
+	}
+}
+
+func TestContainsNilSlice(t *testing.T) {
+	tests := []struct {
+		slice         []string
+		containsValue string
+		expected      bool
+	}{
+		{slice: nil, containsValue: "cat", expected: false},
+		{slice: nil, containsValue: "dog", expected: false},
+	}
+
+	for _, test := range tests {
+		actual := mizu.Contains(test.slice, test.containsValue)
+		if actual != test.expected {
+			t.Errorf("unexpected result - expected: %v, actual: %v", test.expected, actual)
+		}
+	}
+}

shared/go.mod

@@ -3,8 +3,7 @@ module github.com/up9inc/mizu/shared
 go 1.16
 
 require (
-	github.com/google/martian v2.1.0+incompatible // indirect
-	github.com/gorilla/websocket v1.4.2
-	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect
 	github.com/docker/go-units v0.4.0
+	github.com/gorilla/websocket v1.4.2
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

shared/go.sum

@@ -1,8 +1,8 @@
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
-github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

shared/models.go

@@ -75,9 +75,9 @@ func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSoc
 }
 
 type TrafficFilteringOptions struct {
-	PlainTextMaskingRegexes []*SerializableRegexp
-	HideHealthChecks        bool
-	DisableRedaction        bool
+	HealthChecksUserAgentHeaders []string
+	PlainTextMaskingRegexes      []*SerializableRegexp
+	DisableRedaction             bool
 }
 
 type VersionResponse struct {

tap/passive_tapper.go

@@ -84,7 +84,6 @@ var staleTimeoutSeconds = flag.Int("staletimout", 120, "Max time in seconds to k
 var memprofile = flag.String("memprofile", "", "Write memory profile")
 
 // output
-var dumpToHar = flag.Bool("hardump", false, "Dump traffic to har files")
 var HarOutputDir = flag.String("hardir", "", "Directory in which to store output har files")
 var harEntriesPerFile = flag.Int("harentriesperfile", 200, "Number of max number of har entries to store in each file")
 
@@ -186,19 +185,12 @@ func (c *Context) GetCaptureInfo() gopacket.CaptureInfo {
 func StartPassiveTapper(opts *TapOpts) (<-chan *OutputChannelItem, <-chan *OutboundLink) {
 	hostMode = opts.HostMode
 
-	var harWriter *HarWriter
-	if *dumpToHar {
-		harWriter = NewHarWriter(*HarOutputDir, *harEntriesPerFile)
-	}
+	harWriter := NewHarWriter(*HarOutputDir, *harEntriesPerFile)
 	outboundLinkWriter := NewOutboundLinkWriter()
 
 	go startPassiveTapper(harWriter, outboundLinkWriter)
 
-	if harWriter != nil {
-		return harWriter.OutChan, outboundLinkWriter.OutChan
-	}
-
-	return nil, outboundLinkWriter.OutChan
+	return harWriter.OutChan, outboundLinkWriter.OutChan
 }
 
 func startMemoryProfiler() {
@@ -321,10 +313,8 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 		}
 	}
 
-	if *dumpToHar {
-		harWriter.Start()
-		defer harWriter.Stop()
-	}
+	harWriter.Start()
+	defer harWriter.Stop()
 	defer outboundLinkWriter.Stop()
 
 	var dec gopacket.Decoder