Add to periodic stats print in tapper (#221)

#patch

PERMISSIONS.md

@@ -0,0 +1,328 @@
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+# Kubernetes permissions for MIZU  
+
+This document describes in details all permissions required for full and correct operation of Mizu
+
+We broke down this list into few categories:
+- Required - what is needed for `mizu` to run properly on your k8s cluster
+- Optional - permissions needed for proper name resolving for service & pod IPs 
+   - addition required for policy validation 
+ 
+
+
+# Required permissions
+
+Mizu needs following permissions on your Kubernetes cluster to run properly
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+```
+
+## Permissions required for service / pod name resolving (opt)
+
+Optionally, for proper resolving of IP addresses to Kubernetes service name, Mizu needs below permissions:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterroles
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterrolebindings
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  - apps
+  - extensions
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+```
+
+## Permissions for Policy rules validation feature (opt)
+
+Optionally, in order to use the policy rules validation feature, Mizu requires the following additional permissions:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - create
+  - delete
+```
+
+- - -
+
+## Namespace-Restricted mode
+
+Alternatively, in order to restrict Mizu to one namespace only (by setting `agent.namespace` in the config file), Mizu needs the following permissions in that namespace:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+```
+
+### Name resolving in Namespace-Restricted mode (opt)
+
+To restrict Mizu to one namespace while also resolving IPs, Mizu needs the following permissions in that namespace:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  - apps
+  - extensions
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+```

README.md

@@ -39,317 +39,15 @@ Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page.
 
 ## Prerequisites
 1. Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config`
-2. Mizu needs following permissions on your Kubernetes cluster to run
+2. `mizu` assumes user running the command has permissions to create resources (such as pods, services, namespaces) on your Kubernetes cluster (no worries - `mizu` resources are cleaned up upon termination)
 
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - create
-  - delete
-- apiGroups:
-  - apps
-  resources:
-  - daemonsets
-  verbs:
-  - create
-  - patch
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - namespaces
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services/proxy
-  verbs:
-  - get
-```
+For detailed list of k8s permissions see [PERMISSIONS](PERMISSIONS.md) document
 
-3. Optionally, for resolving traffic IP to Kubernetes service name, Mizu needs below permissions
-
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - apps
-  resources:
-  - daemonsets
-  verbs:
-  - create
-  - patch
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - namespaces
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services/proxy
-  verbs:
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - serviceaccounts
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - clusterroles
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - clusterrolebindings
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - roles
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - rolebindings
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - apps
-  - extensions
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - apps
-  - extensions
-  resources:
-  - services
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  - apps
-  - extensions
-  resources:
-  - endpoints
-  verbs:
-  - get
-  - list
-  - watch
-```
-
-4. Optionally, in order to use the policy rules validation feature, Mizu requires the following additional permissions:
-
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - get
-  - create
-  - delete
-```
-
-5. Alternatively, in order to restrict Mizu to one namespace only (by setting `agent.namespace` in the config file), Mizu needs the following permissions in that namespace:
-
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - apps
-  resources:
-  - daemonsets
-  verbs:
-  - get
-  - create
-  - patch
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services/proxy
-  verbs:
-  - get
-```
-
-6. To restrict Mizu to one namespace while also resolving IPs, Mizu needs the following permissions in that namespace:
-
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - apps
-  resources:
-  - daemonsets
-  verbs:
-  - get
-  - create
-  - patch
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services/proxy
-  verbs:
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - serviceaccounts
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - roles
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - rolebindings
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - apps
-  - extensions
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - apps
-  - extensions
-  resources:
-  - services
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  - apps
-  - extensions
-  resources:
-  - endpoints
-  verbs:
-  - get
-  - list
-  - watch
-```
-
-See `examples/roles` for example `clusterroles`. 
 
 ## How to Run
 
 1. Find pods you'd like to tap to in your Kubernetes cluster
-2. Run `mizu tap PODNAME` or `mizu tap REGEX` 
+2. Run `mizu tap` or `mizu tap PODNAME`  
 3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI .. 
 4. Watch the API traffic flowing ..
 5. Type ^C to stop
@@ -358,6 +56,23 @@ See `examples/roles` for example `clusterroles`.
 
 Run `mizu help` for usage options
 
+To tap all pods in current namespace - 
+``` 
+ $ kubectl get pods 
+ NAME                            READY   STATUS    RESTARTS   AGE
+ carts-66c77f5fbb-fq65r          2/2     Running   0          20m
+ catalogue-5f4cb7cf5-7zrmn       2/2     Running   0          20m
+ front-end-649fc5fd6-kqbtn       2/2     Running   0          20m
+ ..
+
+ $ mizu tap
+ +carts-66c77f5fbb-fq65r
+ +catalogue-5f4cb7cf5-7zrmn
+ +front-end-649fc5fd6-kqbtn
+ Web interface is now available at http://localhost:8899
+ ^C
+```
+
 
 To tap specific pod - 
 ``` 
@@ -388,6 +103,22 @@ To tap multiple pods using regex -
  ^C
 ```
 
+## Configuration
+
+Mizu can work with config file which should be stored in ${HOME}/.mizu/config.yaml (macOS: ~/.mizu/config.yaml) <br />
+In case no config file found, defaults will be used. <br />
+In case of partial configuration defined, all other fields will be used with defaults. <br />
+You can always override the defaults or config file with CLI flags.
+
+To get the default config params run `mizu config` <br />
+To generate a new config file with default values use `mizu config -r`
+
+Mizu has several undocumented flags which can be set by using --set flag (e.g., `mizu tap --set dump-logs=true`)
+* **mizu-resources-namespace**: Type - String, See [Namespace-Restricted Mode](#namespace-restricted-mode)
+* **telemetry**: Type - Boolean, Reports telemetry
+* **dump-logs**: Type - Boolean, At the end of the execution it creates a zip file with logs (in .mizu folder)
+* **kube-config-path**: Type - String, Setting the path to kube config (which isn't in standard path)
+
 ## Advanced Usage
 
 ### Namespace-Restricted Mode

agent/pkg/controllers/entries_controller.go

@@ -57,7 +57,7 @@ func GetEntries(c *gin.Context) {
 }
 
 func GetHARs(c *gin.Context) {
-	entriesFilter := &models.HarFetchRequestBody{}
+	entriesFilter := &models.HarFetchRequestQuery{}
 	order := database.OrderDesc
 	if err := c.BindQuery(entriesFilter); err != nil {
 		c.JSON(http.StatusBadRequest, err)
@@ -146,12 +146,12 @@ func GetHARs(c *gin.Context) {
 func UploadEntries(c *gin.Context) {
 	rlog.Infof("Upload entries - started\n")
 
-	uploadRequestBody := &models.UploadEntriesRequestBody{}
-	if err := c.BindQuery(uploadRequestBody); err != nil {
+	uploadParams := &models.UploadEntriesRequestQuery{}
+	if err := c.BindQuery(uploadParams); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
-	if err := validation.Validate(uploadRequestBody); err != nil {
+	if err := validation.Validate(uploadParams); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
@@ -160,19 +160,19 @@ func UploadEntries(c *gin.Context) {
 		return
 	}
 
-	rlog.Infof("Upload entries - creating token. dest %s\n", uploadRequestBody.Dest)
-	token, err := up9.CreateAnonymousToken(uploadRequestBody.Dest)
+	rlog.Infof("Upload entries - creating token. dest %s\n", uploadParams.Dest)
+	token, err := up9.CreateAnonymousToken(uploadParams.Dest)
 	if err != nil {
 		c.String(http.StatusServiceUnavailable, "Cannot analyze, mizu is already analyzing")
 		return
 	}
 	rlog.Infof("Upload entries - uploading. token: %s model: %s\n", token.Token, token.Model)
-	go up9.UploadEntriesImpl(token.Token, token.Model, uploadRequestBody.Dest, uploadRequestBody.SleepIntervalSec)
+	go up9.UploadEntriesImpl(token.Token, token.Model, uploadParams.Dest, uploadParams.SleepIntervalSec)
 	c.String(http.StatusOK, "OK")
 }
 
 func GetFullEntries(c *gin.Context) {
-	entriesFilter := &models.HarFetchRequestBody{}
+	entriesFilter := &models.HarFetchRequestQuery{}
 	if err := c.BindQuery(entriesFilter); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 	}

agent/pkg/models/models.go

@@ -119,19 +119,19 @@ func (fedex *FullEntryDetailsExtra) UnmarshalData(entry *MizuEntry) error {
 }
 
 type EntriesFilter struct {
-	Limit     int    `query:"limit" validate:"required,min=1,max=200"`
-	Operator  string `query:"operator" validate:"required,oneof='lt' 'gt'"`
-	Timestamp int64  `query:"timestamp" validate:"required,min=1"`
+	Limit     int    `form:"limit" validate:"required,min=1,max=200"`
+	Operator  string `form:"operator" validate:"required,oneof='lt' 'gt'"`
+	Timestamp int64  `form:"timestamp" validate:"required,min=1"`
 }
 
-type UploadEntriesRequestBody struct {
+type UploadEntriesRequestQuery struct {
 	Dest             string `form:"dest"`
 	SleepIntervalSec int    `form:"interval"`
 }
 
-type HarFetchRequestBody struct {
-	From int64 `query:"from"`
-	To   int64 `query:"to"`
+type HarFetchRequestQuery struct {
+	From int64 `form:"from"`
+	To   int64 `form:"to"`
 }
 
 type WebSocketEntryMessage struct {

cli/README.md

@@ -1,26 +0,0 @@
-# mizu CLI
-## Usage
-`./mizu {pod_name_regex}`
-
-### Optional Flags
-
-| flag                 | default          | purpose                                                                                                      |
-|----------------------|------------------|--------------------------------------------------------------------------------------------------------------|
-| `--no-gui`           | `false`          | Don't host the web interface (not applicable at the moment)                                                      |
-| `--gui-port`         | `8899`           | local port that web interface will be forwarded to                                                               |
-| `--namespace`        |                  | use namespace different than the one found in kubeconfig                                                     |
-| `--kubeconfig`       |                  | Path to custom kubeconfig file                                                                               |
-
-There are some extra flags defined in code that will show up in `./mizu --help`, these are non functional stubs for now
-
-## Installation
-Make sure your go version is at least 1.11
-1. cd to `mizu/cli`
-2. Run `go mod download` (may take a moment)
-3. Run `go build mizu.go`
-
-Alternatively, you can build+run directly using `go run mizu.go {pod_name_regex}`
-
-
-## Known issues
-* mid-flight port forwarding failures are not detected and no indication will be shown when this occurs

cli/cmd/fetch.go

@@ -12,7 +12,7 @@ var fetchCmd = &cobra.Command{
 	Short: "Download recorded traffic to files",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		go mizu.ReportRun("fetch", mizu.Config.Fetch)
-		if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.Fetch.MizuPort); err != nil {
+		if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.Fetch.GuiPort); err != nil {
 			return err
 		} else if !isCompatible {
 			return nil
@@ -31,5 +31,5 @@ func init() {
 	fetchCmd.Flags().StringP(configStructs.DirectoryFetchName, "d", defaultFetchConfig.Directory, "Provide a custom directory for fetched entries")
 	fetchCmd.Flags().Int(configStructs.FromTimestampFetchName, defaultFetchConfig.FromTimestamp, "Custom start timestamp for fetched entries")
 	fetchCmd.Flags().Int(configStructs.ToTimestampFetchName, defaultFetchConfig.ToTimestamp, "Custom end timestamp fetched entries")
-	fetchCmd.Flags().Uint16P(configStructs.MizuPortFetchName, "p", defaultFetchConfig.MizuPort, "Custom port for mizu")
+	fetchCmd.Flags().Uint16P(configStructs.GuiPortFetchName, "p", defaultFetchConfig.GuiPort, "Provide a custom port for the web interface webserver")
 }

cli/cmd/fetchRunner.go

@@ -16,7 +16,7 @@ import (
 )
 
 func RunMizuFetch() {
-	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Fetch.MizuPort)
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Fetch.GuiPort)
 	resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, mizu.Config.Fetch.FromTimestamp, mizu.Config.Fetch.ToTimestamp))
 	if err != nil {
 		log.Fatal(err)

cli/cmd/tap.go

@@ -64,9 +64,9 @@ func init() {
 	tapCmd.Flags().Bool(configStructs.AnalysisTapName, defaultTapConfig.Analysis, "Uploads traffic to UP9 for further analysis (Beta)")
 	tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
 	tapCmd.Flags().StringArrayP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
-	tapCmd.Flags().Bool(configStructs.HideHealthChecksTapName, defaultTapConfig.HideHealthChecks, "hides requests with kube-probe or prometheus user-agent headers")
+	tapCmd.Flags().Bool(configStructs.HideHealthChecksTapName, defaultTapConfig.HideHealthChecks, "Hides requests with kube-probe or prometheus user-agent headers")
 	tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
-	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "override the default max entries db size of 200mb")
+	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
 	tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any")
 	tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them")
 	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file with policy rules")

cli/mizu/config.go

@@ -69,6 +69,10 @@ func GetConfigWithDefaults() (string, error) {
 	if err := defaults.Set(&defaultConf); err != nil {
 		return "", err
 	}
+
+	// TODO: change to generic solution
+	defaultConf.AgentImage = ""
+
 	return uiUtils.PrettyYaml(defaultConf)
 }
 
@@ -129,6 +133,7 @@ func mergeSetFlag(setValues []string) {
 
 		if !Contains(allowedSetFlags, argumentKey) {
 			Log.Warningf(uiUtils.Warning, fmt.Sprintf("Ignoring set argument %s, flag name must be one of the following: \"%s\"", setValue, strings.Join(allowedSetFlags, "\", \"")))
+			continue
 		}
 
 		mergeFlagValue(configElem, argumentKey, argumentValue)

cli/mizu/configStruct.go

@@ -19,7 +19,7 @@ type ConfigStruct struct {
 	Fetch                  configStructs.FetchConfig   `yaml:"fetch"`
 	Version                configStructs.VersionConfig `yaml:"version"`
 	View                   configStructs.ViewConfig    `yaml:"view"`
-	AgentImage             string                      `yaml:"agent-image"`
+	AgentImage             string                      `yaml:"agent-image,omitempty"`
 	MizuResourcesNamespace string                      `yaml:"mizu-resources-namespace" default:"mizu"`
 	Telemetry              bool                        `yaml:"telemetry" default:"true"`
 	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`

cli/mizu/configStructs/fetchConfig.go

@@ -4,12 +4,12 @@ const (
 	DirectoryFetchName     = "directory"
 	FromTimestampFetchName = "from"
 	ToTimestampFetchName   = "to"
-	MizuPortFetchName      = "port"
+	GuiPortFetchName       = "gui-port"
 )
 
 type FetchConfig struct {
 	Directory     string `yaml:"directory" default:"."`
 	FromTimestamp int    `yaml:"from" default:"0"`
 	ToTimestamp   int    `yaml:"to" default:"0"`
-	MizuPort      uint16 `yaml:"port" default:"8899"`
+	GuiPort       uint16 `yaml:"gui-port" default:"8899"`
 }

tap/http_reader.go

@@ -79,6 +79,7 @@ func (h *httpReader) Read(p []byte) (int, error) {
 			clientHello := tlsx.ClientHello{}
 			err := clientHello.Unmarshall(msg.bytes)
 			if err == nil {
+				statsTracker.incTlsConnectionsCount()
 				fmt.Printf("Detected TLS client hello with SNI %s\n", clientHello.SNI)
 				numericPort, _ := strconv.Atoi(h.tcpID.dstPort)
 				h.outboundLinkWriter.WriteOutboundLink(h.tcpID.srcIP, h.tcpID.dstIP, numericPort, clientHello.SNI, TLSProtocol)
@@ -176,7 +177,7 @@ func (h *httpReader) handleHTTP2Stream() error {
 	}
 
 	if reqResPair != nil {
-		statsTracker.incMatchedMessages()
+		statsTracker.incMatchedPairs()
 
 		if h.harWriter != nil {
 			h.harWriter.WritePair(
@@ -215,7 +216,7 @@ func (h *httpReader) handleHTTP1ClientStream(b *bufio.Reader) error {
 	ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.srcIP, h.tcpID.dstIP, h.tcpID.srcPort, h.tcpID.dstPort, h.messageCount)
 	reqResPair := reqResMatcher.registerRequest(ident, req, h.captureTime)
 	if reqResPair != nil {
-		statsTracker.incMatchedMessages()
+		statsTracker.incMatchedPairs()
 
 		if h.harWriter != nil {
 			h.harWriter.WritePair(
@@ -281,7 +282,7 @@ func (h *httpReader) handleHTTP1ServerStream(b *bufio.Reader) error {
 	ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.dstIP, h.tcpID.srcIP, h.tcpID.dstPort, h.tcpID.srcPort, h.messageCount)
 	reqResPair := reqResMatcher.registerResponse(ident, res, h.captureTime)
 	if reqResPair != nil {
-		statsTracker.incMatchedMessages()
+		statsTracker.incMatchedPairs()
 
 		if h.harWriter != nil {
 			h.harWriter.WritePair(

tap/passive_tapper.go

@@ -10,9 +10,9 @@ package tap
 
 import (
 	"encoding/hex"
+	"encoding/json"
 	"flag"
 	"fmt"
-	"github.com/romana/rlog"
 	"log"
 	"os"
 	"os/signal"
@@ -23,6 +23,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/romana/rlog"
+
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/examples/util"
 	"github.com/google/gopacket/ip4defrag"
@@ -384,9 +386,7 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 			errorMapLen := len(errorsMap)
 			errorsSummery := fmt.Sprintf("%v", errorsMap)
 			errorsMapMutex.Unlock()
-			log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v) - Errors Summary: %s",
-				statsTracker.appStats.TotalPacketsCount,
-				statsTracker.appStats.TotalProcessedBytes,
+			log.Printf("%v (errors: %v, errTypes:%v) - Errors Summary: %s",
 				time.Since(statsTracker.appStats.StartTime),
 				nErrors,
 				errorMapLen,
@@ -405,14 +405,15 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 
 			// Since the last print
 			cleanStats := cleaner.dumpStats()
-			matchedMessages := statsTracker.dumpStats()
 			log.Printf(
-				"flushed connections %d, closed connections: %d, deleted messages: %d, matched messages: %d",
+				"cleaner - flushed connections: %d, closed connections: %d, deleted messages: %d",
 				cleanStats.flushed,
 				cleanStats.closed,
 				cleanStats.deleted,
-				matchedMessages,
 			)
+			currentAppStats := statsTracker.dumpStats()
+			appStatsJSON, _ := json.Marshal(currentAppStats)
+			log.Printf("app stats - %v", string(appStatsJSON))
 		}
 	}()
 
@@ -424,7 +425,7 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 		packetsCount := statsTracker.incPacketsCount()
 		rlog.Debugf("PACKET #%d", packetsCount)
 		data := packet.Data()
-		statsTracker.updateProcessedSize(int64(len(data)))
+		statsTracker.updateProcessedBytes(int64(len(data)))
 		if *hexdumppkt {
 			rlog.Debugf("Packet content (%d/0x%x) - %s", len(data), len(data), hex.Dump(data))
 		}
@@ -458,6 +459,7 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 
 		tcp := packet.Layer(layers.LayerTypeTCP)
 		if tcp != nil {
+			statsTracker.incTcpPacketsCount()
 			tcp := tcp.(*layers.TCP)
 			if *checksum {
 				err := tcp.SetNetworkLayerForChecksum(packet.NetworkLayer())
@@ -475,14 +477,14 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 			assemblerMutex.Unlock()
 		}
 
-		done := *maxcount > 0 && statsTracker.appStats.TotalPacketsCount >= *maxcount
+		done := *maxcount > 0 && statsTracker.appStats.PacketsCount >= *maxcount
 		if done {
 			errorsMapMutex.Lock()
 			errorMapLen := len(errorsMap)
 			errorsMapMutex.Unlock()
 			log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)",
-				statsTracker.appStats.TotalPacketsCount,
-				statsTracker.appStats.TotalProcessedBytes,
+				statsTracker.appStats.PacketsCount,
+				statsTracker.appStats.ProcessedBytes,
 				time.Since(statsTracker.appStats.StartTime),
 				nErrors,
 				errorMapLen)

tap/stats_tracker.go

@@ -6,50 +6,99 @@ import (
 )
 
 type AppStats struct {
-	StartTime            time.Time `json:"startTime"`
-	MatchedMessages      int       `json:"matchedMessages"`
-	TotalPacketsCount    int64     `json:"totalPacketsCount"`
-	TotalProcessedBytes  int64     `json:"totalProcessedBytes"`
-	TotalMatchedMessages int64     `json:"totalMatchedMessages"`
+	StartTime                   time.Time `json:"-"`
+	ProcessedBytes              int64     `json:"processedBytes"`
+	PacketsCount                int64     `json:"packetsCount"`
+	TcpPacketsCount             int64     `json:"tcpPacketsCount"`
+	ReassembledTcpPayloadsCount int64     `json:"reassembledTcpPayloadsCount"`
+	TlsConnectionsCount         int64     `json:"tlsConnectionsCount"`
+	MatchedPairs                int64     `json:"matchedPairs"`
 }
 
 type StatsTracker struct {
-	appStats                AppStats
-	matchedMessagesMutex    sync.Mutex
-	totalPacketsCountMutex  sync.Mutex
-	totalProcessedSizeMutex sync.Mutex
+	appStats                         AppStats
+	processedBytesMutex              sync.Mutex
+	packetsCountMutex                sync.Mutex
+	tcpPacketsCountMutex             sync.Mutex
+	reassembledTcpPayloadsCountMutex sync.Mutex
+	tlsConnectionsCountMutex         sync.Mutex
+	matchedPairsMutex                sync.Mutex
 }
 
-func (st *StatsTracker) incMatchedMessages() {
-	st.matchedMessagesMutex.Lock()
-	st.appStats.MatchedMessages++
-	st.appStats.TotalMatchedMessages++
-	st.matchedMessagesMutex.Unlock()
+func (st *StatsTracker) incMatchedPairs() {
+	st.matchedPairsMutex.Lock()
+	st.appStats.MatchedPairs++
+	st.matchedPairsMutex.Unlock()
 }
 
 func (st *StatsTracker) incPacketsCount() int64 {
-	st.totalPacketsCountMutex.Lock()
-	st.appStats.TotalPacketsCount++
-	currentPacketsCount := st.appStats.TotalPacketsCount
-	st.totalPacketsCountMutex.Unlock()
+	st.packetsCountMutex.Lock()
+	st.appStats.PacketsCount++
+	currentPacketsCount := st.appStats.PacketsCount
+	st.packetsCountMutex.Unlock()
 	return currentPacketsCount
 }
 
-func (st *StatsTracker) updateProcessedSize(size int64) {
-	st.totalProcessedSizeMutex.Lock()
-	st.appStats.TotalProcessedBytes += size
-	st.totalProcessedSizeMutex.Unlock()
+func (st *StatsTracker) incTcpPacketsCount() {
+	st.tcpPacketsCountMutex.Lock()
+	st.appStats.TcpPacketsCount++
+	st.tcpPacketsCountMutex.Unlock()
+}
+
+func (st *StatsTracker) incReassembledTcpPayloadsCount() {
+	st.reassembledTcpPayloadsCountMutex.Lock()
+	st.appStats.ReassembledTcpPayloadsCount++
+	st.reassembledTcpPayloadsCountMutex.Unlock()
+}
+
+func (st *StatsTracker) incTlsConnectionsCount() {
+	st.tlsConnectionsCountMutex.Lock()
+	st.appStats.TlsConnectionsCount++
+	st.tlsConnectionsCountMutex.Unlock()
+}
+
+func (st *StatsTracker) updateProcessedBytes(size int64) {
+	st.processedBytesMutex.Lock()
+	st.appStats.ProcessedBytes += size
+	st.processedBytesMutex.Unlock()
 }
 
 func (st *StatsTracker) setStartTime(startTime time.Time) {
 	st.appStats.StartTime = startTime
 }
 
-func (st *StatsTracker) dumpStats() int {
-	st.matchedMessagesMutex.Lock()
-	matchedMessages := st.appStats.MatchedMessages
-	st.appStats.MatchedMessages = 0
-	st.matchedMessagesMutex.Unlock()
+func (st *StatsTracker) dumpStats() *AppStats {
+	currentAppStats := &AppStats{StartTime: st.appStats.StartTime}
 
-	return matchedMessages
+	st.processedBytesMutex.Lock()
+	currentAppStats.ProcessedBytes = st.appStats.ProcessedBytes
+	st.appStats.ProcessedBytes = 0
+	st.processedBytesMutex.Unlock()
+
+	st.packetsCountMutex.Lock()
+	currentAppStats.PacketsCount = st.appStats.PacketsCount
+	st.appStats.PacketsCount = 0
+	st.packetsCountMutex.Unlock()
+
+	st.tcpPacketsCountMutex.Lock()
+	currentAppStats.TcpPacketsCount = st.appStats.TcpPacketsCount
+	st.appStats.TcpPacketsCount = 0
+	st.tcpPacketsCountMutex.Unlock()
+
+	st.reassembledTcpPayloadsCountMutex.Lock()
+	currentAppStats.ReassembledTcpPayloadsCount = st.appStats.ReassembledTcpPayloadsCount
+	st.appStats.ReassembledTcpPayloadsCount = 0
+	st.reassembledTcpPayloadsCountMutex.Unlock()
+
+	st.tlsConnectionsCountMutex.Lock()
+	currentAppStats.TlsConnectionsCount = st.appStats.TlsConnectionsCount
+	st.appStats.TlsConnectionsCount = 0
+	st.tlsConnectionsCountMutex.Unlock()
+
+	st.matchedPairsMutex.Lock()
+	currentAppStats.MatchedPairs = st.appStats.MatchedPairs
+	st.appStats.MatchedPairs = 0
+	st.matchedPairsMutex.Unlock()
+
+	return currentAppStats
 }

tap/tcp_stream.go

@@ -148,6 +148,7 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass
 			}
 			// This is where we pass the reassembled information onwards
 			// This channel is read by an httpReader object
+			statsTracker.incReassembledTcpPayloadsCount()
 			if dir == reassembly.TCPDirClientToServer && !t.reversed {
 				t.client.msgQueue <- httpReaderDataMsg{data, ac.GetCaptureInfo().Timestamp}
 			} else {