Adding Snyk to CI (#414)

* debug mizu

* Update clean.go, cleanRunner.go, and tapRunner.go

* Update tapRunner.go

* Update cleanRunner.go

* Update tapRunner.go

Co-authored-by: Rami Berman <rami.berman@up9.com>

.github/workflows/security_validation.yml

@@ -0,0 +1,42 @@
+name: Security validation
+
+on:
+  pull_request:
+    branches:
+      - 'develop'
+      - 'main'
+
+jobs:
+  security:
+    name: Check for vulnerabilities
+    runs-on: ubuntu-latest
+    env:
+      SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: snyk/actions/setup@master
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
+
+      - name: CLI
+        working-directory: ./cli
+        run: snyk test
+
+      - name: Agent
+        working-directory: ./agent
+        run: snyk test
+
+      - name: Shared
+        working-directory: ./shared
+        run: snyk test
+
+      - name: Tap
+        working-directory: ./tap
+        run: snyk test
+
+      - name: UI
+        working-directory: ./ui
+        run: snyk test

Dockerfile

@@ -44,7 +44,7 @@ RUN go build -ldflags="-s -w \
 COPY devops/build_extensions.sh ..
 RUN cd .. && /bin/bash build_extensions.sh
 
-FROM alpine:3.13.5
+FROM alpine:3.14
 
 RUN apk add bash libpcap-dev tcpdump
 WORKDIR /app

Makefile

@@ -46,6 +46,10 @@ push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
 	devops/build-push-featurebranch.sh
 
+push-docker-debug:
+	@echo "publishing debug Docker image .. "
+	devops/build-push-featurebranch-debug.sh
+
 build-docker-ci: ## Build agent docker image for CI.
 	@echo "building docker image for ci"
 	devops/build-agent-ci.sh

README.md

@@ -171,6 +171,16 @@ against the contracts.
 
 Please see [CONTRACT MONITORING](docs/CONTRACT_MONITORING.md) page for more details and syntax.
 
+### Configure proxy host 
+
+By default, mizu will be accessible via local host: 'http://localhost:8899/mizu/', it is possible to change the host,
+for instance, to '0.0.0.0' which can grant access via machine IP address.
+This setting can be changed via command line flag `--set tap.proxy-host=<value>` or via config file:
+tap
+    proxy-host: 0.0.0.0
+and when changed it will support accessing by IP
+
+
 ## How to Run local UI
 
 - run from mizu/agent `go run main.go --hars-read --hars-dir <folder>`

agent/pkg/database/main.go

@@ -13,11 +13,12 @@ import (
 )
 
 const (
-	DBPath    = "./entries.db"
-	OrderDesc = "desc"
-	OrderAsc  = "asc"
-	LT        = "lt"
-	GT        = "gt"
+	DBPath     = "./entries.db"
+	OrderDesc  = "desc"
+	OrderAsc   = "asc"
+	LT         = "lt"
+	GT         = "gt"
+	TimeFormat = "2006-01-02 15:04:05.000000000"
 )
 
 var (
@@ -57,7 +58,7 @@ func initDataBase(databasePath string) *gorm.DB {
 	return temp
 }
 
-func GetEntriesFromDb(timestampFrom int64, timestampTo int64, protocolName *string) []tapApi.MizuEntry {
+func GetEntriesFromDb(timeFrom time.Time, timeTo time.Time, protocolName *string) []tapApi.MizuEntry {
 	order := OrderDesc
 	protocolNameCondition := "1 = 1"
 	if protocolName != nil {
@@ -67,7 +68,7 @@ func GetEntriesFromDb(timestampFrom int64, timestampTo int64, protocolName *stri
 	var entries []tapApi.MizuEntry
 	GetEntriesTable().
 		Where(protocolNameCondition).
-		Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)).
+		Where(fmt.Sprintf("created_at BETWEEN '%s' AND '%s'", timeFrom.Format(TimeFormat), timeTo.Format(TimeFormat))).
 		Order(fmt.Sprintf("timestamp %s", order)).
 		Find(&entries)
 

agent/pkg/up9/main.go

@@ -206,13 +206,13 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 
 	sleepTime := time.Second * time.Duration(uploadIntervalSec)
 
-	var timestampFrom int64 = 0
+	var timeFrom time.Time
+	protocolFilter := "http"
 
 	for {
-		timestampTo := time.Now().UnixNano() / int64(time.Millisecond)
-		logger.Log.Infof("Getting entries from %v, to %v\n", timestampFrom, timestampTo)
-		protocolFilter := "http"
-		entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo, &protocolFilter)
+		timeTo := time.Now()
+		logger.Log.Infof("Getting entries from %v, to %v\n", timeFrom.Format(time.RFC3339Nano), timeTo.Format(time.RFC3339Nano))
+		entriesArray := database.GetEntriesFromDb(timeFrom, timeTo, &protocolFilter)
 
 		if len(entriesArray) > 0 {
 			result := make([]har.Entry, 0)
@@ -276,13 +276,14 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 			analyzeInformation.SentCount += len(entriesArray)
 			logger.Log.Infof("Finish uploading %v entries to %s\n", len(entriesArray), GetTrafficDumpUrl(envPrefix, model))
 
+			logger.Log.Infof("Uploaded %v entries until now", analyzeInformation.SentCount)
 		} else {
 			logger.Log.Infof("Nothing to upload")
 		}
 
 		logger.Log.Infof("Sleeping for %v...\n", sleepTime)
 		time.Sleep(sleepTime)
-		timestampFrom = timestampTo
+		timeFrom = timeTo
 	}
 }
 

cli/auth/authProvider.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"net"
 	"net/http"
-	"os"
 	"time"
 
 	"github.com/google/uuid"
@@ -33,19 +32,8 @@ func Login() error {
 		Token:   token.AccessToken,
 	}
 
-	configFile, defaultConfigErr := config.GetConfigWithDefaults()
-	if defaultConfigErr != nil {
-		return fmt.Errorf("failed getting config with defaults, err: %v", defaultConfigErr)
-	}
-
-	if err := config.LoadConfigFile(config.Config.ConfigFilePath, configFile); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("failed getting config file, err: %v", err)
-	}
-
-	configFile.Auth = authConfig
-
-	if err := config.WriteConfig(configFile); err != nil {
-		return fmt.Errorf("failed writing config with auth, err: %v", err)
+	if err := config.UpdateConfig(func(configStruct *config.ConfigStruct) { configStruct.Auth = authConfig }); err != nil {
+		return fmt.Errorf("failed updating config with auth, err: %v", err)
 	}
 
 	config.Config.Auth = authConfig

cli/cmd/clean.go

@@ -0,0 +1,20 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/telemetry"
+)
+
+var cleanCmd = &cobra.Command{
+	Use:   "clean",
+	Short: "Removes all mizu resources",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("clean", nil)
+		performCleanCommand()
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(cleanCmd)
+}

cli/cmd/cleanRunner.go

@@ -0,0 +1,17 @@
+package cmd
+
+import (
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
+)
+
+func performCleanCommand() {
+	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath())
+	if err != nil {
+		logger.Log.Error(err)
+		return
+	}
+
+	finishMizuExecution(kubernetesProvider)
+}

cli/cmd/common.go

@@ -21,7 +21,7 @@ func GetApiServerUrl() string {
 }
 
 func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
-	err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
+	err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.ProxyHost, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
 	if err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v\n"+
 			"Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName))

cli/cmd/tap.go

@@ -80,10 +80,19 @@ Supported protocols are HTTP and gRPC.`,
 
 func askConfirmation(flagName string) {
 	logger.Log.Infof(fmt.Sprintf(uploadTrafficMessageToConfirm, flagName))
+
+	if !config.Config.Tap.AskUploadConfirmation {
+		return
+	}
+
 	if !uiUtils.AskForConfirmation("Would you like to proceed [Y/n]: ") {
 		logger.Log.Infof("You can always run mizu without %s, aborting", flagName)
 		os.Exit(0)
 	}
+
+	if err := config.UpdateConfig(func(configStruct *config.ConfigStruct) { configStruct.Tap.AskUploadConfirmation = false }); err != nil {
+		logger.Log.Debugf("failed updating config with upload confirmation, err: %v", err)
+	}
 }
 
 func init() {

cli/cmd/tapRunner.go

@@ -2,8 +2,11 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/ioutil"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"path"
 	"regexp"
 	"strings"
@@ -129,11 +132,18 @@ func RunMizuTap() {
 		return
 	}
 
-	defer finishMizuExecution(kubernetesProvider)
 	if err := createMizuResources(ctx, kubernetesProvider, mizuValidationRules, contract); err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
+
+		var statusError *k8serrors.StatusError
+		if errors.As(err, &statusError) {
+			if statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists {
+				logger.Log.Info("Mizu is already running in this namespace, change the `mizu-resources-namespace` configuration or run `mizu clean` to remove the currently running Mizu instance")
+			}
+		}
 		return
 	}
+	defer finishMizuExecution(kubernetesProvider)
 
 	go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel, mizuApiFilteringOptions)
 	go goUtils.HandleExcWrapper(watchTapperPod, ctx, kubernetesProvider, cancel)

cli/config/config.go

@@ -40,7 +40,7 @@ func InitConfig(cmd *cobra.Command) error {
 
 	configFilePathFlag := cmd.Flags().Lookup(ConfigFilePathCommandName)
 	configFilePath := configFilePathFlag.Value.String()
-	if err := LoadConfigFile(configFilePath, &Config); err != nil {
+	if err := loadConfigFile(configFilePath, &Config); err != nil {
 		if configFilePathFlag.Changed || !os.IsNotExist(err) {
 			return fmt.Errorf("invalid config, %w\n"+
 				"you can regenerate the file by removing it (%v) and using `mizu config -r`", err, configFilePath)
@@ -81,7 +81,27 @@ func WriteConfig(config *ConfigStruct) error {
 	return nil
 }
 
-func LoadConfigFile(configFilePath string, config *ConfigStruct) error {
+type updateConfigStruct func(*ConfigStruct)
+func UpdateConfig(updateConfigStruct updateConfigStruct) error {
+	configFile, err := GetConfigWithDefaults()
+	if err != nil {
+		return fmt.Errorf("failed getting config with defaults, err: %v", err)
+	}
+
+	if err := loadConfigFile(Config.ConfigFilePath, configFile); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed getting config file, err: %v", err)
+	}
+
+	updateConfigStruct(configFile)
+
+	if err := WriteConfig(configFile); err != nil {
+		return fmt.Errorf("failed writing config, err: %v", err)
+	}
+
+	return nil
+}
+
+func loadConfigFile(configFilePath string, config *ConfigStruct) error {
 	reader, openErr := os.Open(configFilePath)
 	if openErr != nil {
 		return openErr

cli/config/configStructs/tapConfig.go

@@ -26,6 +26,7 @@ type TapConfig struct {
 	UploadIntervalSec      int       `yaml:"upload-interval" default:"10"`
 	PodRegexStr            string    `yaml:"regex" default:".*"`
 	GuiPort                uint16    `yaml:"gui-port" default:"8899"`
+	ProxyHost              string    `yaml:"proxy-host" default:"127.0.0.1"`
 	Namespaces             []string  `yaml:"namespaces"`
 	Analysis               bool      `yaml:"analysis" default:"false"`
 	AllNamespaces          bool      `yaml:"all-namespaces" default:"false"`
@@ -37,6 +38,7 @@ type TapConfig struct {
 	Workspace              string    `yaml:"workspace"`
 	EnforcePolicyFile      string    `yaml:"traffic-validation-file"`
 	ContractFile           string    `yaml:"contract"`
+	AskUploadConfirmation  bool      `yaml:"ask-upload-confirmation" default:"true"`
 	ApiServerResources     Resources `yaml:"api-server-resources"`
 	TapperResources        Resources `yaml:"tapper-resources"`
 }

cli/kubernetes/provider.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/up9inc/mizu/shared/semver"
 	"k8s.io/apimachinery/pkg/version"
 	"net/url"
 	"path/filepath"
@@ -83,6 +84,10 @@ func NewProvider(kubeConfigPath string) (*Provider, error) {
 		return nil, err
 	}
 
+	if err := validateKubernetesVersion(clientSet); err != nil {
+		return nil, err
+	}
+
 	return &Provider{
 		clientSet:        clientSet,
 		kubernetesConfig: kubernetesConfig,
@@ -761,3 +766,19 @@ func validateNotProxy(kubernetesConfig clientcmd.ClientConfig, restClientConfig
 
 	return nil
 }
+
+func validateKubernetesVersion(clientSet *kubernetes.Clientset) error {
+	serverVersion, err := clientSet.ServerVersion()
+	if err != nil {
+		logger.Log.Debugf("error while getting kubernetes server version, err: %v", err)
+		return nil
+	}
+
+	serverVersionSemVer := semver.SemVersion(serverVersion.GitVersion)
+	minKubernetesServerVersionSemVer := semver.SemVersion(mizu.MinKubernetesServerVersion)
+	if minKubernetesServerVersionSemVer.GreaterThan(serverVersionSemVer) {
+		return fmt.Errorf("kubernetes server version %v is not supported, supporting only kubernetes server version of %v or higher", serverVersion.GitVersion, mizu.MinKubernetesServerVersion)
+	}
+
+	return nil
+}

cli/kubernetes/proxy.go

@@ -14,12 +14,12 @@ import (
 const k8sProxyApiPrefix = "/"
 const mizuServicePort = 80
 
-func StartProxy(kubernetesProvider *Provider, mizuPort uint16, mizuNamespace string, mizuServiceName string) error {
+func StartProxy(kubernetesProvider *Provider, proxyHost string, mizuPort uint16, mizuNamespace string, mizuServiceName string) error {
 	logger.Log.Debugf("Starting proxy. namespace: [%v], service name: [%s], port: [%v]", mizuNamespace, mizuServiceName, mizuPort)
 	filter := &proxy.FilterServer{
 		AcceptPaths:   proxy.MakeRegexpArrayOrDie(proxy.DefaultPathAcceptRE),
 		RejectPaths:   proxy.MakeRegexpArrayOrDie(proxy.DefaultPathRejectRE),
-		AcceptHosts:   proxy.MakeRegexpArrayOrDie(proxy.DefaultHostAcceptRE),
+		AcceptHosts:   proxy.MakeRegexpArrayOrDie("^.*"),
 		RejectMethods: proxy.MakeRegexpArrayOrDie(proxy.DefaultMethodRejectRE),
 	}
 
@@ -32,7 +32,7 @@ func StartProxy(kubernetesProvider *Provider, mizuPort uint16, mizuNamespace str
 	mux.Handle("/static/", getRerouteHttpHandlerMizuStatic(proxyHandler, mizuNamespace, mizuServiceName))
 	mux.Handle("/mizu/", getRerouteHttpHandlerMizuAPI(proxyHandler, mizuNamespace, mizuServiceName))
 
-	l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", "127.0.0.1", int(mizuPort)))
+	l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", proxyHost, int(mizuPort)))
 	if err != nil {
 		return err
 	}

cli/mizu/consts.go

@@ -14,17 +14,18 @@ var (
 )
 
 const (
-	MizuResourcesPrefix    = "mizu-"
-	ApiServerPodName       = MizuResourcesPrefix + "api-server"
-	ClusterRoleBindingName = MizuResourcesPrefix + "cluster-role-binding"
-	ClusterRoleName        = MizuResourcesPrefix + "cluster-role"
-	K8sAllNamespaces       = ""
-	RoleBindingName        = MizuResourcesPrefix + "role-binding"
-	RoleName               = MizuResourcesPrefix + "role"
-	ServiceAccountName     = MizuResourcesPrefix + "service-account"
-	TapperDaemonSetName    = MizuResourcesPrefix + "tapper-daemon-set"
-	TapperPodName          = MizuResourcesPrefix + "tapper"
-	ConfigMapName          = MizuResourcesPrefix + "policy"
+	MizuResourcesPrefix        = "mizu-"
+	ApiServerPodName           = MizuResourcesPrefix + "api-server"
+	ClusterRoleBindingName     = MizuResourcesPrefix + "cluster-role-binding"
+	ClusterRoleName            = MizuResourcesPrefix + "cluster-role"
+	K8sAllNamespaces           = ""
+	RoleBindingName            = MizuResourcesPrefix + "role-binding"
+	RoleName                   = MizuResourcesPrefix + "role"
+	ServiceAccountName         = MizuResourcesPrefix + "service-account"
+	TapperDaemonSetName        = MizuResourcesPrefix + "tapper-daemon-set"
+	TapperPodName              = MizuResourcesPrefix + "tapper"
+	ConfigMapName              = MizuResourcesPrefix + "policy"
+	MinKubernetesServerVersion = "1.16.0"
 )
 
 func GetMizuFolderPath() string {

debug.Dockerfile

@@ -12,7 +12,7 @@ FROM golang:1.16-alpine AS builder
 # Set necessary environment variables needed for our image.
 ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 
-RUN apk add libpcap-dev gcc g++ make
+RUN apk add libpcap-dev gcc g++ make bash
 
 # Move to agent working directory (/agent-build).
 WORKDIR /app/agent-build
@@ -20,17 +20,25 @@ WORKDIR /app/agent-build
 COPY agent/go.mod agent/go.sum ./
 COPY shared/go.mod shared/go.mod ../shared/
 COPY tap/go.mod tap/go.mod ../tap/
-
+COPY tap/api/go.* ../tap/api/
 RUN go mod download
 # cheap trick to make the build faster (As long as go.mod wasn't changes)
 RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
 
+ARG COMMIT_HASH
+ARG GIT_BRANCH
+ARG BUILD_TIMESTAMP
+ARG SEM_VER
+
 # Copy and build agent code
 COPY shared ../shared
 COPY tap ../tap
 COPY agent .
 RUN go build -gcflags="all=-N -l" -o mizuagent .
 
+COPY devops/build_extensions_debug.sh ..
+RUN cd .. && /bin/bash build_extensions_debug.sh
+
 
 FROM golang:1.16-alpine
 
@@ -39,6 +47,7 @@ WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
+COPY --from=builder ["/app/agent/build/extensions", "extensions"]
 COPY --from=site-build ["/app/ui-build/build", "site"]
 
 # install remote debugging tool

devops/build-push-featurebranch-debug.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+GCP_PROJECT=up9-docker-hub
+REPOSITORY=gcr.io/$GCP_PROJECT
+SERVER_NAME=mizu
+GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
+
+DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
+SEM_VER=${SEM_VER=0.0.0}
+
+DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
+
+if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
+then
+  echo "Pushing to $GIT_BRANCH is allowed only via CI"
+  exit 1
+fi
+
+echo "building ${DOCKER_TAGGED_BUILDS[@]}"
+DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
+docker build -f debug.Dockerfile $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
+
+for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
+do
+  echo pushing "$DOCKER_TAG"
+  docker push "$DOCKER_TAG"
+done

devops/build_extensions_debug.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+for f in tap/extensions/*; do
+    if [ -d "$f" ]; then
+        extension=$(basename $f) && \
+        cd tap/extensions/${extension} && \
+        go build -gcflags="all=-N -l" -buildmode=plugin -o ../${extension}.so .  && \
+        cd ../../..  && \
+        mkdir -p agent/build/extensions  && \
+        cp tap/extensions/${extension}.so agent/build/extensions
+    fi
+done