Adding logs and fixing several issues (#162)

* Config grooming and several general fixes

Makefile

@@ -19,15 +19,16 @@ help: ## This help.
 TS_SUFFIX="$(shell date '+%s')"
 GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
 BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
+export SEM_VER?=0.0.0
 
-ui: ## build UI
+ui: ## Build UI.
 	@(cd ui; npm i ; npm run build; )
 	@ls -l ui/build  
 
-cli: # build CLI
+cli: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build
 
-agent: ## build mizuagent server
+agent: ## Build agent.
 	@(echo "building mizu agent .." )
 	@(cd agent; go build -o build/mizuagent main.go)
 	@ls -l agent/build
@@ -36,17 +37,17 @@ agent: ## build mizuagent server
 #	@(cd tap; go build -o build/tap ./src)
 #	@ls -l tap/build
 
-docker: ## build Docker image 
+docker: ## Build Docker image.
 	@(echo "building docker image" )
 	./build-push-featurebranch.sh
 
-push: push-docker push-cli ## build and publish Mizu docker image & CLI
+push: push-docker push-cli ## Build and publish agent docker image & CLI.
 
-push-docker: 
+push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
 	./build-push-featurebranch.sh
 
-push-cli:
+push-cli: ## Build and publish CLI.
 	@echo "publishing CLI .. "
 	@cd cli; $(MAKE) build-all
 	@echo "publishing file ${OUTPUT_FILE} .."
@@ -55,17 +56,17 @@ push-cli:
 	gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
 
 
-clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts
+clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
 
-clean-ui: 
+clean-ui: ## Clean UI.
 	@(rm -rf ui/build ; echo "UI cleanup done" )
 
-clean-agent: 
+clean-agent: ## Clean agent.
 	@(rm -rf agent/build ; echo "agent cleanup done" )
 
-clean-cli: 
+clean-cli:  ## Clean CLI.
 	@(cd cli; make clean ; echo "CLI cleanup done" )
 
-clean-docker: 
+clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 

agent/main.go

@@ -85,7 +85,7 @@ func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) {
 	app := gin.Default()
 
 	app.GET("/echo", func(c *gin.Context) {
-		c.String(http.StatusOK, "Hello, World 👋!")
+		c.String(http.StatusOK, "Here is Mizu agent")
 	})
 
 	eventHandlers := api.RoutesEventHandlers{

build-push-featurebranch.sh

@@ -5,7 +5,9 @@ SERVER_NAME=mizu
 GCP_PROJECT=up9-docker-hub
 REPOSITORY=gcr.io/$GCP_PROJECT
 GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
-DOCKER_TAGGED_BUILD=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH:latest
+SEM_VER=${SEM_VER=0.0.0}
+DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
+DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
 
 if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
 then
@@ -13,8 +15,12 @@ then
   exit 1
 fi
 
-echo "building $DOCKER_TAGGED_BUILD"
-docker build -t "$DOCKER_TAGGED_BUILD" --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
+echo "building ${DOCKER_TAGGED_BUILDS[@]}"
+DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
+docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
 
-echo pushing to "$REPOSITORY"
-docker push "$DOCKER_TAGGED_BUILD"
+for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
+do
+        echo pushing "$DOCKER_TAG"
+        docker push "$DOCKER_TAG"
+done

cli/Makefile

@@ -3,6 +3,7 @@ COMMIT_HASH=$(shell git rev-parse HEAD)
 GIT_BRANCH=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
 GIT_VERSION=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
 BUILD_TIMESTAMP=$(shell date +%s)
+export SEM_VER?=0.0.0
 
 .PHONY: help
 .DEFAULT_GOAL := help
@@ -13,7 +14,7 @@ help: ## This help.
 install:
 	go install mizu.go
 
-build:	## build mizu CLI binary (select platform via GOOS / GOARCH env variables)
+build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables).
 	go build -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
@@ -21,7 +22,7 @@ build:	## build mizu CLI binary (select platform via GOOS / GOARCH env variables
 					   -o bin/mizu_$(SUFFIX) mizu.go 
 	(cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256)
 
-build-all:  ## build for all supported platforms
+build-all: ## Build for all supported platforms.
 	@echo "Compiling for every OS and Platform"
 	@mkdir -p bin && echo "SHA256 checksums available for compiled binaries \n\nRun \`shasum -a 256 -c mizu_OS_ARCH.sha256\` to verify\n\n" >  bin/README.md
 	@$(MAKE) build GOOS=darwin GOARCH=amd64
@@ -35,6 +36,6 @@ build-all:  ## build for all supported platforms
 	@echo "---------"
 	@find ./bin -ls
 
-clean: ## clean all build artifacts
+clean: ## Clean all build artifacts.
 	go clean
 	rm -rf ./bin/*

cli/cmd/config.go

@@ -8,17 +8,24 @@ import (
 	"io/ioutil"
 )
 
-var outputFileName string
+var regenerateFile bool
 
 var configCmd = &cobra.Command{
 	Use:   "config",
-	Short: "Generate example config file to stdout",
+	Short: "Generate config with default values",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		template := mizu.GetTemplateConfig()
-		if outputFileName != "" {
+		template, err := mizu.GetConfigWithDefaults()
+		if err != nil {
+			mizu.Log.Errorf("Failed generating config with defaults %v", err)
+			return nil
+		}
+		if regenerateFile {
 			data := []byte(template)
-			_ = ioutil.WriteFile(outputFileName, data, 0644)
-			mizu.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, outputFileName)))
+			if err := ioutil.WriteFile(mizu.GetConfigFilePath(), data, 0644); err != nil {
+				mizu.Log.Errorf("Failed writing config %v", err)
+				return nil
+			}
+			mizu.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, mizu.GetConfigFilePath())))
 		} else {
 			mizu.Log.Debugf("Writing template config.\n%v", template)
 			fmt.Printf("%v", template)
@@ -29,6 +36,5 @@ var configCmd = &cobra.Command{
 
 func init() {
 	rootCmd.AddCommand(configCmd)
-
-	configCmd.Flags().StringVarP(&outputFileName, "file", "f", "", "Save content to local file")
+	configCmd.Flags().BoolVarP(&regenerateFile, "regenerate", "r", false, fmt.Sprintf("Regenerate the config file with default values %s", mizu.GetConfigFilePath()))
 }

cli/cmd/fetch.go

@@ -1,30 +1,23 @@
 package cmd
 
 import (
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/configStructs"
 )
 
-type MizuFetchOptions struct {
-	FromTimestamp int64
-	ToTimestamp   int64
-	Directory     string
-	MizuPort      uint16
-}
-
-var mizuFetchOptions = MizuFetchOptions{}
-
 var fetchCmd = &cobra.Command{
 	Use:   "fetch",
 	Short: "Download recorded traffic to files",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go mizu.ReportRun("fetch", mizuTapOptions)
-		if isCompatible, err := mizu.CheckVersionCompatibility(mizuFetchOptions.MizuPort); err != nil {
+		go mizu.ReportRun("fetch", mizu.Config.Fetch)
+		if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.Fetch.MizuPort); err != nil {
 			return err
 		} else if !isCompatible {
 			return nil
 		}
-		RunMizuFetch(&mizuFetchOptions)
+		RunMizuFetch()
 		return nil
 	},
 }
@@ -32,8 +25,11 @@ var fetchCmd = &cobra.Command{
 func init() {
 	rootCmd.AddCommand(fetchCmd)
 
-	fetchCmd.Flags().StringVarP(&mizuFetchOptions.Directory, "directory", "d", ".", "Provide a custom directory for fetched entries")
-	fetchCmd.Flags().Int64Var(&mizuFetchOptions.FromTimestamp, "from", 0, "Custom start timestamp for fetched entries")
-	fetchCmd.Flags().Int64Var(&mizuFetchOptions.ToTimestamp, "to", 0, "Custom end timestamp fetched entries")
-	fetchCmd.Flags().Uint16VarP(&mizuFetchOptions.MizuPort, "port", "p", 8899, "Custom port for mizu")
+	defaultFetchConfig := configStructs.FetchConfig{}
+	defaults.Set(&defaultFetchConfig)
+
+	fetchCmd.Flags().StringP(configStructs.DirectoryFetchName, "d", defaultFetchConfig.Directory, "Provide a custom directory for fetched entries")
+	fetchCmd.Flags().Int(configStructs.FromTimestampFetchName, defaultFetchConfig.FromTimestamp, "Custom start timestamp for fetched entries")
+	fetchCmd.Flags().Int(configStructs.ToTimestampFetchName, defaultFetchConfig.ToTimestamp, "Custom end timestamp fetched entries")
+	fetchCmd.Flags().Uint16P(configStructs.MizuPortFetchName, "p", defaultFetchConfig.MizuPort, "Custom port for mizu")
 }

cli/cmd/fetchRunner.go

@@ -15,9 +15,9 @@ import (
 	"strings"
 )
 
-func RunMizuFetch(fetch *MizuFetchOptions) {
-	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(fetch.MizuPort)
-	resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, fetch.FromTimestamp, fetch.ToTimestamp))
+func RunMizuFetch() {
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Fetch.MizuPort)
+	resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, mizu.Config.Fetch.FromTimestamp, mizu.Config.Fetch.ToTimestamp))
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -33,8 +33,8 @@ func RunMizuFetch(fetch *MizuFetchOptions) {
 	if err != nil {
 		log.Fatal(err)
 	}
-	_ = Unzip(zipReader, fetch.Directory)
 
+	_ = Unzip(zipReader, mizu.Config.Fetch.Directory)
 }
 
 func Unzip(reader *zip.Reader, dest string) error {

cli/cmd/root.go

@@ -7,26 +7,23 @@ import (
 	"github.com/up9inc/mizu/cli/mizu"
 )
 
-var commandLineFlags []string
-
 var rootCmd = &cobra.Command{
 	Use:   "mizu",
 	Short: "A web traffic viewer for kubernetes",
 	Long: `A web traffic viewer for kubernetes
 Further info is available at https://github.com/up9inc/mizu`,
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-		if err := mizu.InitConfig(commandLineFlags); err != nil {
+		if err := mizu.InitConfig(cmd); err != nil {
 			mizu.Log.Errorf("Invalid config, Exit %s", err)
 			return errors.New(fmt.Sprintf("%v", err))
 		}
-		prettifiedConfig := mizu.GetConfigStr()
-		mizu.Log.Debugf("Final Config: %s", prettifiedConfig)
+
 		return nil
 	},
 }
 
 func init() {
-	rootCmd.PersistentFlags().StringSliceVar(&commandLineFlags, "set", []string{}, "Override values using --set")
+	rootCmd.PersistentFlags().StringSlice(mizu.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", mizu.SetCommandName))
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.

cli/cmd/tap.go

@@ -2,39 +2,14 @@ package cmd
 
 import (
 	"errors"
-	"fmt"
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/configStructs"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared/units"
 	"os"
-	"regexp"
-	"strings"
 )
 
-type MizuTapOptions struct {
-	GuiPort                uint16
-	Namespace              string
-	AllNamespaces          bool
-	Analysis               bool
-	AnalysisDestination    string
-	KubeConfigPath         string
-	MizuImage              string
-	PlainTextFilterRegexes []string
-	TapOutgoing            bool
-	HideHealthChecks       bool
-	MaxEntriesDBSizeBytes  int64
-	SleepIntervalSec       uint16
-	DisableRedaction       bool
-}
-
-var mizuTapOptions = &MizuTapOptions{}
-var direction string
-var humanMaxEntriesDBSize string
-var regex *regexp.Regexp
-
-const maxEntriesDBSizeFlagName = "max-entries-db-size"
-
 const analysisMessageToConfirm = `NOTE: running mizu with --analysis flag will upload recorded traffic for further analysis and enriched presentation options.`
 
 var tapCmd = &cobra.Command{
@@ -43,53 +18,31 @@ var tapCmd = &cobra.Command{
 	Long: `Record the ingoing traffic of a kubernetes pod.
 Supported protocols are HTTP and gRPC.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go mizu.ReportRun("tap", mizuTapOptions)
-		RunMizuTap(regex, mizuTapOptions)
+		go mizu.ReportRun("tap", mizu.Config.Tap)
+		RunMizuTap()
 		return nil
-
 	},
 	PreRunE: func(cmd *cobra.Command, args []string) error {
-		mizu.Log.Debugf("Getting params")
-		mizuTapOptions.AnalysisDestination = mizu.GetString(mizu.ConfigurationKeyAnalyzingDestination)
-		mizuTapOptions.SleepIntervalSec = uint16(mizu.GetInt(mizu.ConfigurationKeyUploadInterval))
-		mizuTapOptions.MizuImage = mizu.GetString(mizu.ConfigurationKeyMizuImage)
-		mizu.Log.Debugf(uiUtils.PrettyJson(mizuTapOptions))
-
-		if len(args) == 0 {
-			return errors.New("POD REGEX argument is required")
+		if len(args) == 1 {
+			mizu.Config.Tap.PodRegexStr = args[0]
 		} else if len(args) > 1 {
 			return errors.New("unexpected number of arguments")
 		}
 
-		var compileErr error
-		regex, compileErr = regexp.Compile(args[0])
-		if compileErr != nil {
-			return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], compileErr))
+		if err := mizu.Config.Tap.Validate(); err != nil {
+			return err
 		}
 
-		var parseHumanDataSizeErr error
-		mizuTapOptions.MaxEntriesDBSizeBytes, parseHumanDataSizeErr = units.HumanReadableToBytes(humanMaxEntriesDBSize)
-		if parseHumanDataSizeErr != nil {
-			return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize))
-		}
-		mizu.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes))
+		mizu.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", mizu.Config.Tap.HumanMaxEntriesDBSize)
 
-		directionLowerCase := strings.ToLower(direction)
-		if directionLowerCase == "any" {
-			mizuTapOptions.TapOutgoing = true
-		} else if directionLowerCase == "in" {
-			mizuTapOptions.TapOutgoing = false
-		} else {
-			return errors.New(fmt.Sprintf("%s is not a valid value for flag --direction. Acceptable values are in/any.", direction))
-		}
-
-		if mizuTapOptions.Analysis {
+		if mizu.Config.Tap.Analysis {
 			mizu.Log.Infof(analysisMessageToConfirm)
 			if !uiUtils.AskForConfirmation("Would you like to proceed [Y/n]: ") {
 				mizu.Log.Infof("You can always run mizu without analysis, aborting")
 				os.Exit(0)
 			}
 		}
+
 		return nil
 	},
 }
@@ -97,14 +50,18 @@ Supported protocols are HTTP and gRPC.`,
 func init() {
 	rootCmd.AddCommand(tapCmd)
 
-	tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
-	tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector")
-	tapCmd.Flags().BoolVar(&mizuTapOptions.Analysis, "analysis", false, "Uploads traffic to UP9 for further analysis (Beta)")
-	tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces")
-	tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file")
-	tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "regex-masking", "r", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies")
-	tapCmd.Flags().StringVarP(&direction, "direction", "", "in", "Record traffic that goes in this direction (relative to the tapped pod): in/any")
-	tapCmd.Flags().BoolVar(&mizuTapOptions.HideHealthChecks, "hide-healthchecks", false, "hides requests with kube-probe or prometheus user-agent headers")
-	tapCmd.Flags().StringVarP(&humanMaxEntriesDBSize, maxEntriesDBSizeFlagName, "", "200MB", "override the default max entries db size of 200mb")
-	tapCmd.Flags().BoolVar(&mizuTapOptions.DisableRedaction, "no-redact", false, "Disables redaction of potentially sensitive request/response headers and body values")
+	defaultTapConfig := configStructs.TapConfig{}
+	defaults.Set(&defaultTapConfig)
+
+	tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
+	tapCmd.Flags().StringP(configStructs.NamespaceTapName, "n", defaultTapConfig.Namespace, "Namespace selector")
+	tapCmd.Flags().Bool(configStructs.AnalysisTapName, defaultTapConfig.Analysis, "Uploads traffic to UP9 for further analysis (Beta)")
+	tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
+	tapCmd.Flags().StringP(configStructs.KubeConfigPathTapName, "k", defaultTapConfig.KubeConfigPath, "Path to kube-config file")
+	tapCmd.Flags().StringArrayP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
+	tapCmd.Flags().Bool(configStructs.HideHealthChecksTapName, defaultTapConfig.HideHealthChecks, "hides requests with kube-probe or prometheus user-agent headers")
+	tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
+	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "override the default max entries db size of 200mb")
+	tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any")
+	tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them")
 }

cli/cmd/tapRunner.go

@@ -11,7 +11,6 @@ import (
 	core "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/tools/clientcmd"
-	"log"
 	"net/http"
 	"net/url"
 	"os"
@@ -31,13 +30,13 @@ const (
 
 var currentlyTappedPods []core.Pod
 
-func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
-	mizuApiFilteringOptions, err := getMizuApiFilteringOptions(tappingOptions)
+func RunMizuTap() {
+	mizuApiFilteringOptions, err := getMizuApiFilteringOptions()
 	if err != nil {
 		return
 	}
 
-	kubernetesProvider, err := kubernetes.NewProvider(tappingOptions.KubeConfigPath)
+	kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.Tap.KubeConfigPath)
 	if err != nil {
 		if clientcmd.IsEmptyConfig(err) {
 			mizu.Log.Infof(uiUtils.Red, "Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'\n")
@@ -53,22 +52,21 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel() // cancel will be called when this function exits
 
-	targetNamespace := getNamespace(tappingOptions, kubernetesProvider)
-	if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegexQuery, targetNamespace); err != nil {
-		mizu.Log.Infof("Error listing pods: %v", err)
-		return
-	} else {
-		currentlyTappedPods = matchingPods
-	}
-
+	targetNamespace := getNamespace(kubernetesProvider)
 	var namespacesStr string
 	if targetNamespace != mizu.K8sAllNamespaces {
 		namespacesStr = fmt.Sprintf("namespace \"%s\"", targetNamespace)
 	} else {
 		namespacesStr = "all namespaces"
 	}
+	mizu.CheckNewerVersion()
 	mizu.Log.Infof("Tapping pods in %s", namespacesStr)
 
+	if err, _ := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespace); err != nil {
+		mizu.Log.Infof("Error listing pods: %v", err)
+		return
+	}
+
 	if len(currentlyTappedPods) == 0 {
 		var suggestionStr string
 		if targetNamespace != mizu.K8sAllNamespaces {
@@ -77,34 +75,36 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
 		mizu.Log.Infof("Did not find any pods matching the regex argument%s", suggestionStr)
 	}
 
+	if mizu.Config.Tap.DryRun {
+		return
+	}
+
 	nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods)
 	if err != nil {
 		return
 	}
 
-	if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions, mizuApiFilteringOptions); err != nil {
+	if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions); err != nil {
 		return
 	}
 
-	mizu.CheckNewerVersion()
-	go portForwardApiPod(ctx, kubernetesProvider, cancel, tappingOptions) // TODO convert this to job for built in pod ttl or have the running app handle this
-	go watchPodsForTapping(ctx, kubernetesProvider, cancel, podRegexQuery, tappingOptions)
-	go syncApiStatus(ctx, cancel, tappingOptions)
+	go createProxyToApiServerPod(ctx, kubernetesProvider, cancel)
+	go watchPodsForTapping(ctx, kubernetesProvider, cancel)
 
 	//block until exit signal or error
 	waitForFinish(ctx, cancel)
 }
 
-func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
+func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
 	if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
 		return err
 	}
 
-	if err := createMizuApiServer(ctx, kubernetesProvider, tappingOptions, mizuApiFilteringOptions); err != nil {
+	if err := createMizuApiServer(ctx, kubernetesProvider, mizuApiFilteringOptions); err != nil {
 		return err
 	}
 
-	if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
+	if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
 		return err
 	}
 
@@ -115,12 +115,13 @@ func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Pro
 	_, err := kubernetesProvider.CreateNamespace(ctx, mizu.ResourcesNamespace)
 	if err != nil {
 		mizu.Log.Infof("Error creating Namespace %s: %v", mizu.ResourcesNamespace, err)
+		return err
 	}
-
-	return err
+	mizu.Log.Debugf("Successfully creating Namespace %s", mizu.ResourcesNamespace)
+	return nil
 }
 
-func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
+func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
 	var err error
 
 	mizuServiceAccountExists = createRBACIfNecessary(ctx, kubernetesProvider)
@@ -130,27 +131,30 @@ func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Pro
 	} else {
 		serviceAccountName = ""
 	}
-	_, err = kubernetesProvider.CreateMizuApiServerPod(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName, tappingOptions.MizuImage, serviceAccountName, mizuApiFilteringOptions, tappingOptions.MaxEntriesDBSizeBytes)
+	_, err = kubernetesProvider.CreateMizuApiServerPod(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName, mizu.Config.MizuImage, serviceAccountName, mizuApiFilteringOptions, mizu.Config.Tap.MaxEntriesDBSizeBytes())
 	if err != nil {
 		mizu.Log.Infof("Error creating mizu %s pod: %v", mizu.ApiServerPodName, err)
 		return err
 	}
+	mizu.Log.Debugf("Successfully created API server pod: %s", mizu.ApiServerPodName)
 
 	apiServerService, err = kubernetesProvider.CreateService(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName, mizu.ApiServerPodName)
 	if err != nil {
 		mizu.Log.Infof("Error creating mizu %s service: %v", mizu.ApiServerPodName, err)
 		return err
 	}
+	mizu.Log.Debugf("Successfully created service: %s", mizu.ApiServerPodName)
 
 	return nil
 }
 
-func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.TrafficFilteringOptions, error) {
+func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) {
+
 	var compiledRegexSlice []*shared.SerializableRegexp
 
-	if tappingOptions.PlainTextFilterRegexes != nil && len(tappingOptions.PlainTextFilterRegexes) > 0 {
+	if mizu.Config.Tap.PlainTextFilterRegexes != nil && len(mizu.Config.Tap.PlainTextFilterRegexes) > 0 {
 		compiledRegexSlice = make([]*shared.SerializableRegexp, 0)
-		for _, regexStr := range tappingOptions.PlainTextFilterRegexes {
+		for _, regexStr := range mizu.Config.Tap.PlainTextFilterRegexes {
 			compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr)
 			if err != nil {
 				mizu.Log.Infof("Regex %s is invalid: %v", regexStr, err)
@@ -160,10 +164,10 @@ func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.Traffic
 		}
 	}
 
-	return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: tappingOptions.HideHealthChecks, DisableRedaction: tappingOptions.DisableRedaction}, nil
+	return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: mizu.Config.Tap.HideHealthChecks, DisableRedaction: mizu.Config.Tap.DisableRedaction}, nil
 }
 
-func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error {
+func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string) error {
 	if len(nodeToTappedPodIPMap) > 0 {
 		var serviceAccountName string
 		if mizuServiceAccountExists {
@@ -176,19 +180,20 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
 			ctx,
 			mizu.ResourcesNamespace,
 			mizu.TapperDaemonSetName,
-			tappingOptions.MizuImage,
+			mizu.Config.MizuImage,
 			mizu.TapperPodName,
 			fmt.Sprintf("%s.%s.svc.cluster.local", apiServerService.Name, apiServerService.Namespace),
 			nodeToTappedPodIPMap,
 			serviceAccountName,
-			tappingOptions.TapOutgoing,
+			mizu.Config.Tap.TapOutgoing(),
 		); err != nil {
 			mizu.Log.Infof("Error creating mizu tapper daemonset: %v", err)
 			return err
 		}
+		mizu.Log.Debugf("Successfully created %v tappers", len(nodeToTappedPodIPMap))
 	} else {
 		if err := kubernetesProvider.RemoveDaemonSet(ctx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
-			mizu.Log.Infof("Error deleting mizu tapper daemonset: %v", err)
+			mizu.Log.Errorf("Error deleting mizu tapper daemonset: %v", err)
 			return err
 		}
 	}
@@ -231,27 +236,46 @@ func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
 	}
 }
 
-func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, podRegex *regexp.Regexp, tappingOptions *MizuTapOptions) {
-	targetNamespace := getNamespace(tappingOptions, kubernetesProvider)
-
-	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, targetNamespace), podRegex)
+func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+	targetNamespace := getNamespace(kubernetesProvider)
+	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, targetNamespace), mizu.Config.Tap.PodRegex())
 
+	controlSocketStr := fmt.Sprintf("ws://%s/ws", kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort))
+	controlSocket, err := mizu.CreateControlSocket(controlSocketStr)
+	if err != nil {
+		mizu.Log.Infof("error establishing control socket connection %s", err)
+		cancel()
+	}
+	mizu.Log.Debugf("Control socket created %s", controlSocketStr)
+	err = controlSocket.SendNewTappedPodsListMessage(currentlyTappedPods)
+	if err != nil {
+		mizu.Log.Debugf("error Sending message via control socket %v, error: %s", controlSocketStr, err)
+	}
 	restartTappers := func() {
-		if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegex, targetNamespace); err != nil {
-			mizu.Log.Infof("Error getting pods by regex: %s (%v,%+v)", err, err, err)
+		err, changeFound := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespace)
+		if err != nil {
+			mizu.Log.Errorf("Error getting pods by regex: %s (%v,%+v)", err, err, err)
 			cancel()
-		} else {
-			currentlyTappedPods = matchingPods
+		}
+
+		if !changeFound {
+			mizu.Log.Debugf("Nothing changed update tappers not needed")
+			return
+		}
+
+		err = controlSocket.SendNewTappedPodsListMessage(currentlyTappedPods)
+		if err != nil {
+			mizu.Log.Debugf("error Sending message via control socket %v, error: %s", controlSocketStr, err)
 		}
 
 		nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods)
 		if err != nil {
-			mizu.Log.Infof("Error building node to ips map: %s (%v,%+v)", err, err, err)
+			mizu.Log.Errorf("Error building node to ips map: %s (%v,%+v)", err, err, err)
 			cancel()
 		}
 
-		if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
-			mizu.Log.Infof("Error updating daemonset: %s (%v,%+v)", err, err, err)
+		if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
+			mizu.Log.Errorf("Error updating daemonset: %s (%v,%+v)", err, err, err)
 			cancel()
 		}
 	}
@@ -259,21 +283,21 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 
 	for {
 		select {
-		case newTarget := <-added:
-			mizu.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", newTarget.Name))
-
-		case removedTarget := <-removed:
-			mizu.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s", removedTarget.Name))
+		case pod := <-added:
+			mizu.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace)
 			restartTappersDebouncer.SetOn()
-
-		case modifiedTarget := <-modified:
+		case pod := <-removed:
+			mizu.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace)
+			restartTappersDebouncer.SetOn()
+		case pod := <-modified:
+			mizu.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP)
 			// Act only if the modified pod has already obtained an IP address.
 			// After filtering for IPs, on a normal pod restart this includes the following events:
 			// - Pod deletion
 			// - Pod reaches start state
 			// - Pod reaches ready state
 			// Ready/unready transitions might also trigger this event.
-			if modifiedTarget.Status.PodIP != "" {
+			if pod.Status.PodIP != "" {
 				restartTappersDebouncer.SetOn()
 			}
 
@@ -287,7 +311,53 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 	}
 }
 
-func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
+func updateCurrentlyTappedPods(kubernetesProvider *kubernetes.Provider, ctx context.Context, targetNamespace string) (error, bool) {
+	changeFound := false
+	if matchingPods, err := kubernetesProvider.GetAllRunningPodsMatchingRegex(ctx, mizu.Config.Tap.PodRegex(), targetNamespace); err != nil {
+		mizu.Log.Infof("Error getting pods by regex: %s (%v,%+v)", err, err, err)
+		return err, false
+	} else {
+		addedPods, removedPods := getPodArrayDiff(currentlyTappedPods, matchingPods)
+		for _, addedPod := range addedPods {
+			changeFound = true
+			mizu.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", addedPod.Name))
+		}
+		for _, removedPod := range removedPods {
+			changeFound = true
+			mizu.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s", removedPod.Name))
+		}
+		currentlyTappedPods = matchingPods
+	}
+
+	return nil, changeFound
+}
+
+func getPodArrayDiff(oldPods []core.Pod, newPods []core.Pod) (added []core.Pod, removed []core.Pod) {
+	added = getMissingPods(newPods, oldPods)
+	removed = getMissingPods(oldPods, newPods)
+
+	return added, removed
+}
+
+//returns pods present in pods1 array and missing in pods2 array
+func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod {
+	missingPods := make([]core.Pod, 0)
+	for _, pod1 := range pods1 {
+		var found = false
+		for _, pod2 := range pods2 {
+			if pod1.UID == pod2.UID {
+				found = true
+				break
+			}
+		}
+		if !found {
+			missingPods = append(missingPods, pod1)
+		}
+	}
+	return missingPods
+}
+
+func createProxyToApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.ApiServerPodName))
 	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, mizu.ResourcesNamespace), podExactRegex)
 	isPodReady := false
@@ -296,55 +366,62 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 		select {
 		case <-ctx.Done():
 			return
-
 		case <-added:
+			mizu.Log.Debugf("Got agent pod added event")
 			continue
 		case <-removed:
 			mizu.Log.Infof("%s removed", mizu.ApiServerPodName)
 			cancel()
 			return
 		case modifiedPod := <-modified:
-			if modifiedPod.Status.Phase == "Running" && !isPodReady {
+			mizu.Log.Debugf("Got agent pod modified event, status phase: %v", modifiedPod.Status.Phase)
+			if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
 				isPodReady = true
 				go func() {
-					err := kubernetes.StartProxy(kubernetesProvider, tappingOptions.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName)
+					err := kubernetes.StartProxy(kubernetesProvider, mizu.Config.Tap.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName)
 					if err != nil {
-						mizu.Log.Infof("Error occured while running k8s proxy %v", err)
+						mizu.Log.Errorf("Error occurred while running k8s proxy %v", err)
 						cancel()
 					}
 				}()
-				mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(tappingOptions.GuiPort)
-				mizu.Log.Infof("Mizu is available at http://%s", mizuProxiedUrl)
-
+				mizu.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort))
 				time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready
-				if tappingOptions.Analysis {
-					urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(tappingOptions.AnalysisDestination), tappingOptions.SleepIntervalSec)
-					u, err := url.ParseRequestURI(urlPath)
-
-					if err != nil {
-						log.Fatal(fmt.Sprintf("Failed parsing the URL %v\n", err))
-					}
-					mizu.Log.Debugf("Sending get request to %v", u.String())
-					if response, err := http.Get(u.String()); err != nil || response.StatusCode != 200 {
-						mizu.Log.Infof("error sending upload entries req, status code: %v, err: %v", response.StatusCode, err)
-					} else {
-						mizu.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis")
-					}
-				}
+				requestForAnalysis()
 			}
-
 		case <-timeAfter:
 			if !isPodReady {
-				mizu.Log.Infof("error: %s pod was not ready in time", mizu.ApiServerPodName)
+				mizu.Log.Errorf("Error: %s pod was not ready in time", mizu.ApiServerPodName)
 				cancel()
 			}
-
 		case <-errorChan:
+			mizu.Log.Debugf("[ERROR] Agent creation, watching %v namespace", mizu.ResourcesNamespace)
 			cancel()
 		}
 	}
 }
 
+func requestForAnalysis() {
+	if !mizu.Config.Tap.Analysis {
+		return
+	}
+
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort)
+	urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(mizu.Config.Tap.AnalysisDestination), mizu.Config.Tap.SleepIntervalSec)
+	u, parseErr := url.ParseRequestURI(urlPath)
+	if parseErr != nil {
+		mizu.Log.Fatal("Failed parsing the URL (consider changing the analysis dest URL), err: %v", parseErr)
+	}
+
+	mizu.Log.Debugf("Sending get request to %v", u.String())
+	if response, requestErr := http.Get(u.String()); requestErr != nil {
+		mizu.Log.Errorf("Failed to notify agent for analysis, err: %v", requestErr)
+	} else if response.StatusCode != 200 {
+		mizu.Log.Errorf("Failed to notify agent for analysis, status code: %v", response.StatusCode)
+	} else {
+		mizu.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis")
+	}
+}
+
 func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
 	mizuRBACExists, err := kubernetesProvider.DoesServiceAccountExist(ctx, mizu.ResourcesNamespace, mizu.ServiceAccountName)
 	if err != nil {
@@ -387,34 +464,11 @@ func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
 	}
 }
 
-func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
-	controlSocketStr := fmt.Sprintf("ws://%s/ws", kubernetes.GetMizuApiServerProxiedHostAndPath(tappingOptions.GuiPort))
-	controlSocket, err := mizu.CreateControlSocket(controlSocketStr)
-	if err != nil {
-		mizu.Log.Infof("error establishing control socket connection %s", err)
-		cancel()
-	}
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		default:
-			err = controlSocket.SendNewTappedPodsListMessage(currentlyTappedPods)
-			if err != nil {
-				mizu.Log.Debugf("error Sending message via control socket %v, error: %s", controlSocketStr, err)
-			}
-			time.Sleep(10 * time.Second)
-		}
-	}
-
-}
-
-func getNamespace(tappingOptions *MizuTapOptions, kubernetesProvider *kubernetes.Provider) string {
-	if tappingOptions.AllNamespaces {
+func getNamespace(kubernetesProvider *kubernetes.Provider) string {
+	if mizu.Config.Tap.AllNamespaces {
 		return mizu.K8sAllNamespaces
-	} else if len(tappingOptions.Namespace) > 0 {
-		return tappingOptions.Namespace
+	} else if len(mizu.Config.Tap.Namespace) > 0 {
+		return mizu.Config.Tap.Namespace
 	} else {
 		return kubernetesProvider.CurrentNamespace()
 	}

cli/cmd/version.go

@@ -1,24 +1,20 @@
 package cmd
 
 import (
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/configStructs"
 	"strconv"
 	"time"
 )
 
-type MizuVersionOptions struct {
-	DebugInfo bool
-}
-
-var mizuVersionOptions = &MizuVersionOptions{}
-
 var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print version info",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go mizu.ReportRun("version", mizuVersionOptions)
-		if mizuVersionOptions.DebugInfo {
+		go mizu.ReportRun("version", mizu.Config.Version)
+		if mizu.Config.Version.DebugInfo {
 			timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
 			mizu.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
 			mizu.Log.Infof("Build Time: %s (%s)", mizu.BuildTimestamp, time.Unix(timeStampInt, 0))
@@ -33,6 +29,9 @@ var versionCmd = &cobra.Command{
 func init() {
 	rootCmd.AddCommand(versionCmd)
 
-	versionCmd.Flags().BoolVarP(&mizuVersionOptions.DebugInfo, "debug", "d", false, "Provide all information about version")
+	defaultVersionConfig := configStructs.VersionConfig{}
+	defaults.Set(&defaultVersionConfig)
+
+	versionCmd.Flags().BoolP(configStructs.DebugInfoVersionName, "d", defaultVersionConfig.DebugInfo, "Provide all information about version")
 
 }

cli/cmd/view.go

@@ -1,28 +1,23 @@
 package cmd
 
 import (
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/configStructs"
 )
 
-type MizuViewOptions struct {
-	GuiPort        uint16
-	KubeConfigPath string
-}
-
-var mizuViewOptions = &MizuViewOptions{}
-
 var viewCmd = &cobra.Command{
 	Use:   "view",
 	Short: "Open GUI in browser",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go mizu.ReportRun("view", mizuViewOptions)
-		if isCompatible, err := mizu.CheckVersionCompatibility(mizuViewOptions.GuiPort); err != nil {
+		go mizu.ReportRun("view", mizu.Config.View)
+		if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.View.GuiPort); err != nil {
 			return err
 		} else if !isCompatible {
 			return nil
 		}
-		runMizuView(mizuViewOptions)
+		runMizuView()
 		return nil
 	},
 }
@@ -30,6 +25,9 @@ var viewCmd = &cobra.Command{
 func init() {
 	rootCmd.AddCommand(viewCmd)
 
-	viewCmd.Flags().Uint16VarP(&mizuViewOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
-	viewCmd.Flags().StringVarP(&mizuViewOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file")
+	defaultViewConfig := configStructs.ViewConfig{}
+	defaults.Set(&defaultViewConfig)
+
+	viewCmd.Flags().Uint16P(configStructs.GuiPortViewName, "p", defaultViewConfig.GuiPort, "Provide a custom port for the web interface webserver")
+	viewCmd.Flags().StringP(configStructs.KubeConfigPathViewName, "k", defaultViewConfig.KubeConfigPath, "Path to kube-config file")
 }

cli/cmd/viewRunner.go

@@ -10,8 +10,8 @@ import (
 	"net/http"
 )
 
-func runMizuView(mizuViewOptions *MizuViewOptions) {
-	kubernetesProvider, err := kubernetes.NewProvider(mizuViewOptions.KubeConfigPath)
+func runMizuView() {
+	kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.View.KubeConfigPath)
 	if err != nil {
 		if clientcmd.IsEmptyConfig(err) {
 			mizu.Log.Infof("Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'")
@@ -35,16 +35,16 @@ func runMizuView(mizuViewOptions *MizuViewOptions) {
 		return
 	}
 
-	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizuViewOptions.GuiPort)
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.View.GuiPort)
 	_, err = http.Get(fmt.Sprintf("http://%s/", mizuProxiedUrl))
 	if err == nil {
-		mizu.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, mizuViewOptions.GuiPort)
+		mizu.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, mizu.Config.View.GuiPort)
 		return
 	}
 	mizu.Log.Infof("Found service %s, creating k8s proxy", mizu.ApiServerPodName)
 
-	mizu.Log.Infof("Mizu is available at  http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizuViewOptions.GuiPort))
-	err = kubernetes.StartProxy(kubernetesProvider, mizuViewOptions.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName)
+	mizu.Log.Infof("Mizu is available at  http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.View.GuiPort))
+	err = kubernetes.StartProxy(kubernetesProvider, mizu.Config.View.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName)
 	if err != nil {
 		mizu.Log.Infof("Error occured while running k8s proxy %v", err)
 	}

cli/go.mod

@@ -3,10 +3,12 @@ module github.com/up9inc/mizu/cli
 go 1.16
 
 require (
+	github.com/creasty/defaults v1.5.1
 	github.com/google/go-github/v37 v37.0.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.1.3
+	github.com/spf13/pflag v1.0.5
 	github.com/up9inc/mizu/shared v0.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/api v0.21.2

cli/go.sum

@@ -82,6 +82,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creasty/defaults v1.5.1 h1:j8WexcS3d/t4ZmllX4GEkl4wIB/trOr035ajcLHCISM=
+github.com/creasty/defaults v1.5.1/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

cli/kubernetes/provider.go

@@ -473,6 +473,8 @@ func (provider *Provider) CheckDaemonSetExists(ctx context.Context, namespace st
 }
 
 func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool) error {
+	mizu.Log.Debugf("Applying %d tapper deamonsets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
+
 	if len(nodeToTappedPodIPMap) == 0 {
 		return fmt.Errorf("Daemon set %s must tap at least 1 pod", daemonSetName)
 	}
@@ -493,12 +495,11 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 		mizuCmd = append(mizuCmd, "--anydirection")
 	}
 
-	privileged := true
 	agentContainer := applyconfcore.Container()
 	agentContainer.WithName(tapperPodName)
 	agentContainer.WithImage(podImage)
 	agentContainer.WithImagePullPolicy(core.PullAlways)
-	agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(privileged))
+	agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(true))
 	agentContainer.WithCommand(mizuCmd...)
 	agentContainer.WithEnv(
 		applyconfcore.EnvVar().WithName(shared.HostModeEnvVar).WithValue("1"),
@@ -587,14 +588,14 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	return err
 }
 
-func (provider *Provider) GetAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespace string) ([]core.Pod, error) {
+func (provider *Provider) GetAllRunningPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespace string) ([]core.Pod, error) {
 	pods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
 	if err != nil {
 		return nil, err
 	}
 	matchingPods := make([]core.Pod, 0)
 	for _, pod := range pods.Items {
-		if regex.MatchString(pod.Name) {
+		if regex.MatchString(pod.Name) && isPodRunning(&pod) {
 			matchingPods = append(matchingPods, pod)
 		}
 	}
@@ -635,3 +636,7 @@ func loadKubernetesConfiguration(kubeConfigPath string) clientcmd.ClientConfig {
 		},
 	)
 }
+
+func isPodRunning(pod *core.Pod) bool {
+	return pod.Status.Phase == core.PodRunning
+}

cli/kubernetes/proxy.go

@@ -2,6 +2,7 @@ package kubernetes
 
 import (
 	"fmt"
+	"github.com/up9inc/mizu/cli/mizu"
 	"k8s.io/kubectl/pkg/proxy"
 	"net"
 	"net/http"
@@ -13,6 +14,7 @@ const k8sProxyApiPrefix = "/"
 const mizuServicePort = 80
 
 func StartProxy(kubernetesProvider *Provider, mizuPort uint16, mizuNamespace string, mizuServiceName string) error {
+	mizu.Log.Debugf("Starting proxy. namespace: [%v], service name: [%s], port: [%v]", mizuNamespace, mizuServiceName, mizuPort)
 	filter := &proxy.FilterServer{
 		AcceptPaths:   proxy.MakeRegexpArrayOrDie(proxy.DefaultPathAcceptRE),
 		RejectPaths:   proxy.MakeRegexpArrayOrDie(proxy.DefaultPathRejectRE),

cli/mizu/config.go

@@ -3,6 +3,9 @@ package mizu
 import (
 	"errors"
 	"fmt"
+	"github.com/creasty/defaults"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"gopkg.in/yaml.v3"
 	"io/ioutil"
@@ -13,209 +16,184 @@ import (
 	"strings"
 )
 
-const separator = "="
-
-var configObj = map[string]interface{}{}
-
-type CommandLineFlag struct {
-	CommandLineName   string
-	YamlHierarchyName string
-	DefaultValue      interface{}
-}
-
 const (
-	ConfigurationKeyAnalyzingDestination = "tap.dest"
-	ConfigurationKeyUploadInterval       = "tap.uploadInterval"
-	ConfigurationKeyMizuImage            = "mizuImage"
-	ConfigurationKeyTelemetry            = "telemetry"
+	Separator      = "="
+	SetCommandName = "set"
 )
 
-var allowedSetFlags = []CommandLineFlag{
-	{
-		CommandLineName:   "dest",
-		YamlHierarchyName: ConfigurationKeyAnalyzingDestination,
-		DefaultValue:      "up9.app",
-		// TODO: maybe add short description that we can show
-	},
-	{
-		CommandLineName:   "uploadInterval",
-		YamlHierarchyName: ConfigurationKeyUploadInterval,
-		DefaultValue:      10,
-	},
-	{
-		CommandLineName:   "mizuImage",
-		YamlHierarchyName: ConfigurationKeyMizuImage,
-		DefaultValue:      fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", Branch, SemVer),
-	},
-	{
-		CommandLineName:   "telemetry",
-		YamlHierarchyName: ConfigurationKeyTelemetry,
-		DefaultValue:      true,
-	},
-}
+var Config = ConfigStruct{}
 
-func GetString(key string) string {
-	return fmt.Sprintf("%v", getValueFromMergedConfig(key))
-}
-
-func GetBool(key string) bool {
-	stringVal := GetString(key)
-	Log.Debugf("Found string value %v", stringVal)
-
-	val, err := strconv.ParseBool(stringVal)
-	if err != nil {
-		Log.Warningf(uiUtils.Red, fmt.Sprintf( "Invalid value %v for key %s, expected bool", stringVal, key))
-		os.Exit(1)
+func InitConfig(cmd *cobra.Command) error {
+	if err := defaults.Set(&Config); err != nil {
+		return err
 	}
-	return val
-}
 
-func GetInt(key string) int {
-	stringVal := GetString(key)
-	Log.Debugf("Found string value %v", stringVal)
+	if err := mergeConfigFile(); err != nil {
+		Log.Errorf("Could not load config file, error %v", err)
+		Log.Fatalf("You can regenerate the file using `mizu config -r` or just remove it %v", GetConfigFilePath())
+	}
 
-	val, err := strconv.Atoi(stringVal)
-	if err != nil {
-		Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for key %s, expected int", stringVal, key))
-		os.Exit(1)
-	}
-	return val
-}
+	cmd.Flags().Visit(initFlag)
+
+	finalConfigPrettified, _ := uiUtils.PrettyJson(Config)
+	Log.Debugf("Init config finished\n Final config: %v", finalConfigPrettified)
 
-func InitConfig(commandLineValues []string) error {
-	Log.Debugf("Merging default values")
-	mergeDefaultValues()
-	Log.Debugf("Merging config file values")
-	if err1 := mergeConfigFile(); err1 != nil {
-		Log.Infof(fmt.Sprintf(uiUtils.Red, "Invalid config file\n"))
-		return err1
-	}
-	Log.Debugf("Merging command line values")
-	if err2 := mergeCommandLineFlags(commandLineValues); err2 != nil {
-		Log.Infof(fmt.Sprintf(uiUtils.Red, "Invalid commanad argument\n"))
-		return err2
-	}
-	finalConfigPrettified, _ := uiUtils.PrettyJson(configObj)
-	Log.Debugf("Merged all config successfully\n Final config: %v", finalConfigPrettified)
 	return nil
 }
 
-func GetTemplateConfig() string {
-	templateConfig := map[string]interface{}{}
-	for _, allowedFlag := range allowedSetFlags {
-		addToConfigObj(allowedFlag.YamlHierarchyName, allowedFlag.DefaultValue, templateConfig)
+func GetConfigWithDefaults() (string, error) {
+	defaultConf := ConfigStruct{}
+	if err := defaults.Set(&defaultConf); err != nil {
+		return "", err
 	}
-	prettifiedConfig, _ := uiUtils.PrettyYaml(templateConfig)
-	return prettifiedConfig
+	return uiUtils.PrettyYaml(defaultConf)
 }
 
-func GetConfigStr() string {
-	val, _ := uiUtils.PrettyYaml(configObj)
-	return val
-}
-
-func getValueFromMergedConfig(key string) interface{} {
-	if a, ok := configObj[key]; ok {
-		return a
-	}
-	return nil
-}
-
-func mergeDefaultValues() {
-	for _, allowedFlag := range allowedSetFlags {
-		Log.Debugf("Setting %v to %v", allowedFlag.YamlHierarchyName, allowedFlag.DefaultValue)
-		configObj[allowedFlag.YamlHierarchyName] = allowedFlag.DefaultValue
-	}
+func GetConfigFilePath() string {
+	return path.Join(getMizuFolderPath(), "config.yaml")
 }
 
 func mergeConfigFile() error {
-	Log.Debugf("Merging mizu config file values")
-	home, homeDirErr := os.UserHomeDir()
-	if homeDirErr != nil {
-		return nil
-	}
-	reader, openErr := os.Open(path.Join(home, ".mizu", "config.yaml"))
+	reader, openErr := os.Open(GetConfigFilePath())
 	if openErr != nil {
 		return nil
 	}
+
 	buf, readErr := ioutil.ReadAll(reader)
 	if readErr != nil {
 		return readErr
 	}
-	m := make(map[string]interface{})
-	if err := yaml.Unmarshal(buf, &m); err != nil {
+
+	if err := yaml.Unmarshal(buf, &Config); err != nil {
 		return err
 	}
-	for k, v := range m {
-		addToConfig(k, v)
-	}
+	Log.Debugf("Found config file, merged to default options")
+
 	return nil
 }
 
-func addToConfig(prefix string, value interface{}) {
-	typ := reflect.TypeOf(value).Kind()
-	if typ == reflect.Map {
-		for k1, v1 := range value.(map[string]interface{}) {
-			addToConfig(fmt.Sprintf("%s.%s", prefix, k1), v1)
-		}
-	} else {
-		validateConfigFileKey(prefix)
-		configObj[prefix] = value
+func initFlag(f *pflag.Flag) {
+	configElem := reflect.ValueOf(&Config).Elem()
+
+	sliceValue, isSliceValue := f.Value.(pflag.SliceValue)
+	if !isSliceValue {
+		mergeFlagValue(configElem, f.Name, f.Value.String())
+		return
 	}
+
+	if f.Name == SetCommandName {
+		if setError := mergeSetFlag(sliceValue.GetSlice()); setError != nil {
+			Log.Infof(uiUtils.Red, "Invalid set argument")
+		}
+		return
+	}
+
+	mergeFlagValues(configElem, f.Name, sliceValue.GetSlice())
 }
 
-func mergeCommandLineFlags(commandLineValues []string) error {
-	Log.Debugf("Merging Command line flags")
-	for _, e := range commandLineValues {
-		if !strings.Contains(e, separator) {
-			return errors.New(fmt.Sprintf("invalid set argument %s", e))
+func mergeSetFlag(setValues []string) error {
+	configElem := reflect.ValueOf(&Config).Elem()
+
+	for _, setValue := range setValues {
+		if !strings.Contains(setValue, Separator) {
+			return errors.New(fmt.Sprintf("invalid set argument %s", setValue))
 		}
-		split := strings.SplitN(e, separator, 2)
+
+		split := strings.SplitN(setValue, Separator, 2)
 		if len(split) != 2 {
-			return errors.New(fmt.Sprintf("invalid set argument %s", e))
-		}
-		setFlagKey, argumentValue := split[0], split[1]
-		argumentNameInConfig, err := flagFromAllowed(setFlagKey)
-		if err != nil {
-			return err
+			return errors.New(fmt.Sprintf("invalid set argument %s", setValue))
 		}
 
-		configObj[argumentNameInConfig] = argumentValue
+		argumentKey, argumentValue := split[0], split[1]
+		mergeFlagValue(configElem, argumentKey, argumentValue)
 	}
+
 	return nil
 }
 
-func flagFromAllowed(setFlagKey string) (string, error) {
-	for _, allowedFlag := range allowedSetFlags {
-		if strings.ToLower(allowedFlag.CommandLineName) == strings.ToLower(setFlagKey) {
-			return allowedFlag.YamlHierarchyName, nil
-		}
-	}
-	return "", errors.New(fmt.Sprintf("invalid set argument %s", setFlagKey))
-}
+func mergeFlagValue(currentElem reflect.Value, flagKey string, flagValue string) {
+	for i := 0; i < currentElem.NumField(); i++ {
+		currentField := currentElem.Type().Field(i)
+		currentFieldByName := currentElem.FieldByName(currentField.Name)
 
-func validateConfigFileKey(configFileKey string) {
-	for _, allowedFlag := range allowedSetFlags {
-		if allowedFlag.YamlHierarchyName == configFileKey {
+		if currentField.Type.Kind() == reflect.Struct {
+			mergeFlagValue(currentFieldByName, flagKey, flagValue)
+			continue
+		}
+
+		if currentField.Tag.Get("yaml") != flagKey {
+			continue
+		}
+
+		flagValueKind := currentField.Type.Kind()
+
+		parsedValue, err := getParsedValue(flagValueKind, flagValue)
+		if err != nil {
+			Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for key %s, expected %s", flagValue, flagKey, flagValueKind))
 			return
 		}
+
+		currentFieldByName.Set(parsedValue)
 	}
-	Log.Info(fmt.Sprintf("Unknown argument: %s. Exit", configFileKey))
-	os.Exit(1)
 }
 
-func addToConfigObj(key string, value interface{}, configObj map[string]interface{}) {
-	typ := reflect.TypeOf(value).Kind()
-	if typ != reflect.Map {
-		if strings.Contains(key, ".") {
-			split := strings.SplitN(key, ".", 2)
-			firstLevelKey := split[0]
-			if _, ok := configObj[firstLevelKey]; !ok {
-				configObj[firstLevelKey] = map[string]interface{}{}
-			}
-			addToConfigObj(split[1], value, configObj[firstLevelKey].(map[string]interface{}))
-		} else {
-			configObj[key] = value
+func mergeFlagValues(currentElem reflect.Value, flagKey string, flagValues []string) {
+	for i := 0; i < currentElem.NumField(); i++ {
+		currentField := currentElem.Type().Field(i)
+		currentFieldByName := currentElem.FieldByName(currentField.Name)
+
+		if currentField.Type.Kind() == reflect.Struct {
+			mergeFlagValues(currentFieldByName, flagKey, flagValues)
+			continue
 		}
+
+		if currentField.Tag.Get("yaml") != flagKey {
+			continue
+		}
+
+		flagValueKind := currentField.Type.Elem().Kind()
+
+		parsedValues := reflect.MakeSlice(reflect.SliceOf(currentField.Type.Elem()), 0, 0)
+		for _, flagValue := range flagValues {
+			parsedValue, err := getParsedValue(flagValueKind, flagValue)
+			if err != nil {
+				Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for key %s, expected %s", flagValue, flagKey, flagValueKind))
+				return
+			}
+
+			parsedValues = reflect.Append(parsedValues, parsedValue)
+		}
+
+		currentFieldByName.Set(parsedValues)
 	}
 }
+
+func getParsedValue(kind reflect.Kind, value string) (reflect.Value, error) {
+	switch kind {
+	case reflect.String:
+		return reflect.ValueOf(value), nil
+	case reflect.Bool:
+		boolArgumentValue, err := strconv.ParseBool(value)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(boolArgumentValue), nil
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(intArgumentValue), nil
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uintArgumentValue), nil
+	}
+
+	return reflect.ValueOf(nil), errors.New("value to parse does not match type")
+}

cli/mizu/configStruct.go

@@ -0,0 +1,19 @@
+package mizu
+
+import (
+	"fmt"
+	"github.com/up9inc/mizu/cli/mizu/configStructs"
+)
+
+type ConfigStruct struct {
+	Tap       configStructs.TapConfig     `yaml:"tap"`
+	Fetch     configStructs.FetchConfig   `yaml:"fetch"`
+	Version   configStructs.VersionConfig `yaml:"version"`
+	View      configStructs.ViewConfig    `yaml:"view"`
+	MizuImage string                      `yaml:"mizu-image"`
+	Telemetry bool                        `yaml:"telemetry" default:"true"`
+}
+
+func (config *ConfigStruct) SetDefaults() {
+	config.MizuImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", Branch, SemVer)
+}

cli/mizu/configStructs/fetchConfig.go

@@ -0,0 +1,15 @@
+package configStructs
+
+const (
+	DirectoryFetchName     = "directory"
+	FromTimestampFetchName = "from"
+	ToTimestampFetchName   = "to"
+	MizuPortFetchName      = "port"
+)
+
+type FetchConfig struct {
+	Directory     string `yaml:"directory" default:"."`
+	FromTimestamp int    `yaml:"from" default:"0"`
+	ToTimestamp   int    `yaml:"to" default:"0"`
+	MizuPort      uint16 `yaml:"port" default:"8899"`
+}

cli/mizu/configStructs/tapConfig.go

@@ -0,0 +1,78 @@
+package configStructs
+
+import (
+	"errors"
+	"fmt"
+	"github.com/up9inc/mizu/shared/units"
+	"regexp"
+	"strings"
+)
+
+const (
+	GuiPortTapName                = "gui-port"
+	NamespaceTapName              = "namespace"
+	AnalysisTapName               = "analysis"
+	AllNamespacesTapName          = "all-namespaces"
+	KubeConfigPathTapName         = "kube-config"
+	PlainTextFilterRegexesTapName = "regex-masking"
+	HideHealthChecksTapName       = "hide-healthchecks"
+	DisableRedactionTapName       = "no-redact"
+	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
+	DirectionTapName              = "direction"
+	DryRunTapName                 = "dry-run"
+)
+
+type TapConfig struct {
+	AnalysisDestination    string   `yaml:"dest" default:"up9.app"`
+	SleepIntervalSec       int      `yaml:"upload-interval" default:"10"`
+	PodRegexStr            string   `yaml:"regex" default:".*"`
+	GuiPort                uint16   `yaml:"gui-port" default:"8899"`
+	Namespace              string   `yaml:"namespace"`
+	Analysis               bool     `yaml:"analysis" default:"false"`
+	AllNamespaces          bool     `yaml:"all-namespaces" default:"false"`
+	KubeConfigPath         string   `yaml:"kube-config"`
+	PlainTextFilterRegexes []string `yaml:"regex-masking"`
+	HideHealthChecks       bool     `yaml:"hide-healthchecks" default:"false"`
+	DisableRedaction       bool     `yaml:"no-redact" default:"false"`
+	HumanMaxEntriesDBSize  string   `yaml:"max-entries-db-size" default:"200MB"`
+	Direction              string   `yaml:"direction" default:"in"`
+	DryRun                 bool     `yaml:"dry-run" default:"false"`
+}
+
+func (config *TapConfig) PodRegex() *regexp.Regexp {
+	podRegex, _ := regexp.Compile(config.PodRegexStr)
+	return podRegex
+}
+
+func (config *TapConfig) TapOutgoing() bool {
+	directionLowerCase := strings.ToLower(config.Direction)
+	if directionLowerCase == "any" {
+		return true
+	}
+
+	return false
+}
+
+func (config *TapConfig) MaxEntriesDBSizeBytes() int64 {
+	maxEntriesDBSizeBytes, _ := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
+	return maxEntriesDBSizeBytes
+}
+
+func (config *TapConfig) Validate() error {
+	_, compileErr := regexp.Compile(config.PodRegexStr)
+	if compileErr != nil {
+		return errors.New(fmt.Sprintf("%s is not a valid regex %s", config.PodRegexStr, compileErr))
+	}
+
+	_, parseHumanDataSizeErr := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
+	if parseHumanDataSizeErr != nil {
+		return errors.New(fmt.Sprintf("Could not parse --%s value %s", HumanMaxEntriesDBSizeTapName, config.HumanMaxEntriesDBSize))
+	}
+
+	directionLowerCase := strings.ToLower(config.Direction)
+	if directionLowerCase != "any" && directionLowerCase != "in" {
+		return errors.New(fmt.Sprintf("%s is not a valid value for flag --%s. Acceptable values are in/any.", config.Direction, DirectionTapName))
+	}
+
+	return nil
+}

cli/mizu/configStructs/versionConfig.go

@@ -0,0 +1,9 @@
+package configStructs
+
+const (
+	DebugInfoVersionName = "debug"
+)
+
+type VersionConfig struct {
+	DebugInfo bool `yaml:"debug" default:"false"`
+}

cli/mizu/configStructs/viewConfig.go

@@ -0,0 +1,11 @@
+package configStructs
+
+const (
+	GuiPortViewName        = "gui-port"
+	KubeConfigPathViewName = "kube-config"
+)
+
+type ViewConfig struct {
+	GuiPort        uint16 `yaml:"gui-port" default:"8899"`
+	KubeConfigPath string `yaml:"kube-config"`
+}

cli/mizu/consts.go

@@ -1,5 +1,10 @@
 package mizu
 
+import (
+	"os"
+	"path"
+)
+
 var (
 	SemVer         = "0.0.1"
 	Branch         = "develop"
@@ -18,3 +23,11 @@ const (
 	TapperDaemonSetName    = "mizu-tapper-daemon-set"
 	TapperPodName          = "mizu-tapper"
 )
+
+func getMizuFolderPath() string {
+	home, homeDirErr := os.UserHomeDir()
+	if homeDirErr != nil {
+		return ""
+	}
+	return path.Join(home, ".mizu")
+}

cli/mizu/logger.go

@@ -14,10 +14,9 @@ var format = logging.MustStringFormatter(
 )
 
 func InitLogger() {
-	homeDirPath, _ := os.UserHomeDir()
-	mizuDirPath := path.Join(homeDirPath, ".mizu")
+	mizuDirPath := getMizuFolderPath()
 	if err := os.MkdirAll(mizuDirPath, os.ModePerm); err != nil {
-		panic(fmt.Sprintf("Failed creating .mizu dir: %v, err %v", mizuDirPath, err))
+		panic(fmt.Sprintf("Failed creating mizu dir: %v, err %v", mizuDirPath, err))
 	}
 	logPath := path.Join(mizuDirPath, "log.log")
 	f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
@@ -35,5 +34,6 @@ func InitLogger() {
 
 	logging.SetBackend(backend1Leveled, backend2Formatter)
 
+	Log.Debugf("\n\n\n")
 	Log.Debugf("Running mizu version %v", SemVer)
 }

cli/mizu/telemetry.go

@@ -10,15 +10,11 @@ import (
 const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
 
 func ReportRun(cmd string, args interface{}) {
-	if !GetBool(ConfigurationKeyTelemetry) {
+	if !Config.Telemetry {
 		Log.Debugf("not reporting due to config value")
 		return
 	}
 
-	if Branch != "main" {
-		Log.Debugf("reporting only on main branch")
-		return
-	}
 	argsBytes, _ := json.Marshal(args)
 	argsMap := map[string]string{
 		"telemetry_type": "execution",
@@ -26,6 +22,7 @@ func ReportRun(cmd string, args interface{}) {
 		"args":           string(argsBytes),
 		"component":      "mizu_cli",
 		"BuildTimestamp": BuildTimestamp,
+		"Branch":         Branch,
 		"version":        SemVer}
 	argsMap["message"] = fmt.Sprintf("mizu %v - %v", argsMap["cmd"], string(argsBytes))
 

cli/mizu/versionCheck.go

@@ -55,7 +55,7 @@ func CheckNewerVersion() {
 	client := github.NewClient(nil)
 	latestRelease, _, err := client.Repositories.GetLatestRelease(context.Background(), "up9inc", "mizu")
 	if err != nil {
-		Log.Debugf("Failed to get latest release")
+		Log.Debugf("[ERROR] Failed to get latest release")
 		return
 	}
 
@@ -67,20 +67,20 @@ func CheckNewerVersion() {
 		}
 	}
 	if versionFileUrl == "" {
-		Log.Debugf("Version file not found in the latest release")
+		Log.Debugf("[ERROR] Version file not found in the latest release")
 		return
 	}
 
 	res, err := http.Get(versionFileUrl)
 	if err != nil {
-		Log.Debugf("http.Get version asset -> %v", err)
+		Log.Debugf("[ERROR] Failed to get the version file %v", err)
 		return
 	}
 
 	data, err := ioutil.ReadAll(res.Body)
 	res.Body.Close()
 	if err != nil {
-		Log.Debugf("ioutil.ReadAll -> %v", err)
+		Log.Debugf("[ERROR] Failed to read the version file -> %v", err)
 		return
 	}
 	gitHubVersion := string(data)

tap/cleaner.go

@@ -1,6 +1,7 @@
 package tap
 
 import (
+	"github.com/romana/rlog"
 	"sync"
 	"time"
 
@@ -20,19 +21,21 @@ type Cleaner struct {
 	cleanPeriod       time.Duration
 	connectionTimeout time.Duration
 	stats             CleanerStats
-	statsMutex	  sync.Mutex
+	statsMutex        sync.Mutex
 }
 
 func (cl *Cleaner) clean() {
 	startCleanTime := time.Now()
 
 	cl.assemblerMutex.Lock()
+	rlog.Debugf("Assembler Stats before cleaning %s", cl.assembler.Dump())
 	flushed, closed := cl.assembler.FlushCloseOlderThan(startCleanTime.Add(-cl.connectionTimeout))
 	cl.assemblerMutex.Unlock()
 
 	deleted := cl.matcher.deleteOlderThan(startCleanTime.Add(-cl.connectionTimeout))
 
 	cl.statsMutex.Lock()
+	rlog.Debugf("Assembler Stats after cleaning %s", cl.assembler.Dump())
 	cl.stats.flushed += flushed
 	cl.stats.closed += closed
 	cl.stats.deleted += deleted
@@ -55,7 +58,7 @@ func (cl *Cleaner) dumpStats() CleanerStats {
 
 	stats := CleanerStats{
 		flushed: cl.stats.flushed,
-		closed : cl.stats.closed,
+		closed:  cl.stats.closed,
 		deleted: cl.stats.deleted,
 	}
 

tap/passive_tapper.go

@@ -35,7 +35,8 @@ const AppPortsEnvVar = "APP_PORTS"
 const maxHTTP2DataLenEnvVar = "HTTP2_DATA_SIZE_LIMIT"
 const maxHTTP2DataLenDefault = 1 * 1024 * 1024 // 1MB
 const cleanPeriod = time.Second * 10
-var remoteOnlyOutboundPorts = []int { 80, 443 }
+
+var remoteOnlyOutboundPorts = []int{80, 443}
 
 func parseAppPorts(appPortsList string) []int {
 	ports := make([]int, 0)
@@ -55,10 +56,10 @@ var decoder = flag.String("decoder", "", "Name of the decoder to use (default: g
 var statsevery = flag.Int("stats", 60, "Output statistics every N seconds")
 var lazy = flag.Bool("lazy", false, "If true, do lazy decoding")
 var nodefrag = flag.Bool("nodefrag", false, "If true, do not do IPv4 defrag")
-var checksum = flag.Bool("checksum", false, "Check TCP checksum")  // global
-var nooptcheck = flag.Bool("nooptcheck", true, "Do not check TCP options (useful to ignore MSS on captures with TSO)")  // global
-var ignorefsmerr = flag.Bool("ignorefsmerr", true, "Ignore TCP FSM errors")  // global
-var allowmissinginit = flag.Bool("allowmissinginit", true, "Support streams without SYN/SYN+ACK/ACK sequence")  // global
+var checksum = flag.Bool("checksum", false, "Check TCP checksum")                                                      // global
+var nooptcheck = flag.Bool("nooptcheck", true, "Do not check TCP options (useful to ignore MSS on captures with TSO)") // global
+var ignorefsmerr = flag.Bool("ignorefsmerr", true, "Ignore TCP FSM errors")                                            // global
+var allowmissinginit = flag.Bool("allowmissinginit", true, "Support streams without SYN/SYN+ACK/ACK sequence")         // global
 var verbose = flag.Bool("verbose", false, "Be verbose")
 var debug = flag.Bool("debug", false, "Display debug information")
 var quiet = flag.Bool("quiet", false, "Be quiet regarding errors")
@@ -68,7 +69,7 @@ var nohttp = flag.Bool("nohttp", false, "Disable HTTP parsing")
 var output = flag.String("output", "", "Path to create file for HTTP 200 OK responses")
 var writeincomplete = flag.Bool("writeincomplete", false, "Write incomplete response")
 
-var hexdump = flag.Bool("dump", false, "Dump HTTP request/response as hex")  // global
+var hexdump = flag.Bool("dump", false, "Dump HTTP request/response as hex") // global
 var hexdumppkt = flag.Bool("dumppkt", false, "Dump packet as hex")
 
 // capture
@@ -87,7 +88,7 @@ var dumpToHar = flag.Bool("hardump", false, "Dump traffic to har files")
 var HarOutputDir = flag.String("hardir", "", "Directory in which to store output har files")
 var harEntriesPerFile = flag.Int("harentriesperfile", 200, "Number of max number of har entries to store in each file")
 
-var reqResMatcher = createResponseRequestMatcher()  // global
+var reqResMatcher = createResponseRequestMatcher() // global
 var statsTracker = StatsTracker{}
 
 // global
@@ -117,8 +118,8 @@ var outputLevel int
 var errorsMap map[string]uint
 var errorsMapMutex sync.Mutex
 var nErrors uint
-var ownIps []string           // global
-var hostMode bool             // global
+var ownIps []string // global
+var hostMode bool   // global
 
 /* minOutputLevel: Error will be printed only if outputLevel is above this value
  * t:              key for errorsMap (counting errors)
@@ -196,6 +197,37 @@ func StartPassiveTapper(opts *TapOpts) (<-chan *OutputChannelItem, <-chan *Outbo
 	return nil, outboundLinkWriter.OutChan
 }
 
+func startMemoryProfiler() {
+	dirname := "/app/pprof"
+	rlog.Info("Profiling is on, results will be written to %s", dirname)
+	go func() {
+		if _, err := os.Stat(dirname); os.IsNotExist(err) {
+			if err := os.Mkdir(dirname, 0777); err != nil {
+				log.Fatal("could not create directory for profile: ", err)
+			}
+		}
+
+		for true {
+			t := time.Now()
+
+			filename := fmt.Sprintf("%s/%s__mem.prof", dirname, t.Format("15_04_05"))
+
+			rlog.Info("Writing memory profile to %s\n", filename)
+
+			f, err := os.Create(filename)
+			if err != nil {
+				log.Fatal("could not create memory profile: ", err)
+			}
+			runtime.GC() // get up-to-date statistics
+			if err := pprof.WriteHeapProfile(f); err != nil {
+				log.Fatal("could not write memory profile: ", err)
+			}
+			_ = f.Close()
+			time.Sleep(time.Minute)
+		}
+	}()
+}
+
 func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWriter) {
 	log.SetFlags(log.LstdFlags | log.LUTC | log.Lshortfile)
 
@@ -310,13 +342,19 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 	defragger := ip4defrag.NewIPv4Defragmenter()
 
 	streamFactory := &tcpStreamFactory{
-		doHTTP: !*nohttp,
-		harWriter: harWriter,
+		doHTTP:             !*nohttp,
+		harWriter:          harWriter,
 		outbountLinkWriter: outboundLinkWriter,
-
 	}
 	streamPool := reassembly.NewStreamPool(streamFactory)
 	assembler := reassembly.NewAssembler(streamPool)
+
+	maxBufferedPagesTotal := GetMaxBufferedPagesPerConnection()
+	maxBufferedPagesPerConnection := GetMaxBufferedPagesTotal()
+	rlog.Infof("Assembler options: maxBufferedPagesTotal=%d, maxBufferedPagesPerConnection=%d", maxBufferedPagesTotal, maxBufferedPagesPerConnection)
+	assembler.AssemblerOptions.MaxBufferedPagesTotal = maxBufferedPagesTotal
+	assembler.AssemblerOptions.MaxBufferedPagesPerConnection = maxBufferedPagesPerConnection
+
 	var assemblerMutex sync.Mutex
 
 	signalChan := make(chan os.Signal, 1)
@@ -324,10 +362,10 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 
 	staleConnectionTimeout := time.Second * time.Duration(*staleTimeoutSeconds)
 	cleaner := Cleaner{
-		assembler: assembler,
-		assemblerMutex: &assemblerMutex,
-		matcher: &reqResMatcher,
-		cleanPeriod: cleanPeriod,
+		assembler:         assembler,
+		assemblerMutex:    &assemblerMutex,
+		matcher:           &reqResMatcher,
+		cleanPeriod:       cleanPeriod,
 		connectionTimeout: staleConnectionTimeout,
 	}
 	cleaner.start()
@@ -376,6 +414,10 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 		}
 	}()
 
+	if GetMemoryProfilingEnabled() {
+		startMemoryProfiler()
+	}
+
 	for packet := range source.Packets() {
 		count++
 		rlog.Debugf("PACKET #%d", count)

tap/settings.go

@@ -1,5 +1,18 @@
 package tap
 
+import (
+	"os"
+	"strconv"
+)
+
+const (
+	MemoryProfilingEnabledEnvVarName          = "MEMORY_PROFILING_ENABLED"
+	MaxBufferedPagesTotalEnvVarName           = "MAX_BUFFERED_PAGES_TOTAL"
+	MaxBufferedPagesPerConnectionEnvVarName   = "MAX_BUFFERED_PAGES_PER_CONNECTION"
+	MaxBufferedPagesTotalDefaultValue         = 5000
+	MaxBufferedPagesPerConnectionDefaultValue = 5000
+)
+
 type globalSettings struct {
 	filterPorts       []int
 	filterAuthorities []string
@@ -29,3 +42,23 @@ func GetFilterIPs() []string {
 	copy(addresses, gSettings.filterAuthorities)
 	return addresses
 }
+
+func GetMaxBufferedPagesTotal() int {
+	valueFromEnv, err := strconv.Atoi(os.Getenv(MaxBufferedPagesTotalEnvVarName))
+	if err != nil {
+		return MaxBufferedPagesTotalDefaultValue
+	}
+	return valueFromEnv
+}
+
+func GetMaxBufferedPagesPerConnection() int {
+	valueFromEnv, err := strconv.Atoi(os.Getenv(MaxBufferedPagesPerConnectionEnvVarName))
+	if err != nil {
+		return MaxBufferedPagesPerConnectionDefaultValue
+	}
+	return valueFromEnv
+}
+
+func GetMemoryProfilingEnabled() bool {
+	return os.Getenv(MemoryProfilingEnabledEnvVarName) == "1"
+}