Introducing acceptance test (#222)

.github/workflows/acceptance_tests.yml

@@ -0,0 +1,26 @@
+name: acceptance tests
+on:
+  pull_request:
+    branches:
+      - 'main'
+  push:
+    branches:
+      - 'develop'
+jobs:
+  run-acceptance-tests:
+    name: Run acceptance tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Setup acceptance test
+        run: source ./acceptanceTests/setup.sh
+
+      - name: Test
+        run: make acceptance-test

Makefile

@@ -28,6 +28,9 @@ ui: ## Build UI.
 cli: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build
 
+build-cli-ci: ## Build CLI for CI.
+	@echo "building cli for ci"; cd cli && $(MAKE) build GIT_BRANCH=ci SUFFIX=ci
+
 agent: ## Build agent.
 	@(echo "building mizu agent .." )
 	@(cd agent; go build -o build/mizuagent main.go)
@@ -42,6 +45,10 @@ push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
 	./build-push-featurebranch.sh
 
+build-docker-ci: ## Build agent docker image for CI.
+	@echo "building docker image for ci"
+	./build-agent-ci.sh
+
 push-cli: ## Build and publish CLI.
 	@echo "publishing CLI .. "
 	@cd cli; $(MAKE) build-all
@@ -50,7 +57,6 @@ push-cli: ## Build and publish CLI.
 	gsutil cp -r ./cli/bin/* gs://${BUCKET_PATH}/
 	gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
 
-
 clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
 
 clean-ui: ## Clean UI.
@@ -65,8 +71,11 @@ clean-cli:  ## Clean CLI.
 clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
-test-cli: ## Run tests.
+test-cli:
 	@echo "running cli tests"; cd cli && $(MAKE) test
 
-test-agent: ## Run tests.
+test-agent:
 	@echo "running agent tests"; cd agent && $(MAKE) test
+
+acceptance-test:
+	@echo "running acceptance tests"; cd acceptanceTests && $(MAKE) test

acceptanceTests/Makefile

@@ -0,0 +1,2 @@
+test: ## Run acceptance tests.
+	@go test ./...

acceptanceTests/go.mod

@@ -0,0 +1,3 @@
+module github.com/up9inc/mizu/tests
+
+go 1.16

acceptanceTests/setup.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+PREFIX=$HOME/local/bin
+VERSION=v1.22.0
+
+echo "Attempting to install minikube and assorted tools to $PREFIX"
+
+if ! [ -x "$(command -v kubectl)" ]; then
+  echo "Installing kubectl version $VERSION"
+  curl -LO "https://storage.googleapis.com/kubernetes-release/release/$VERSION/bin/linux/amd64/kubectl"
+  chmod +x kubectl
+  mv kubectl "$PREFIX"
+else
+  echo "kubetcl is already installed"
+fi
+
+if ! [ -x "$(command -v minikube)" ]; then
+  echo "Installing minikube version $VERSION"
+  curl -Lo minikube https://storage.googleapis.com/minikube/releases/$VERSION/minikube-linux-amd64
+  chmod +x minikube
+  mv minikube "$PREFIX"
+else
+  echo "minikube is already installed"
+fi
+
+echo "Starting minikube..."
+minikube start
+
+echo "Creating mizu tests namespace"
+kubectl create namespace mizu-tests
+
+echo "Creating httpbin deployment"
+kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests
+
+echo "Creating httpbin service"
+kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests
+
+echo "Starting proxy"
+kubectl proxy --port=8080 &
+
+echo "Setting minikube docker env"
+eval $(minikube docker-env)
+
+echo "Build agent image"
+make build-docker-ci
+
+echo "Build cli"
+make build-cli-ci

acceptanceTests/tap_test.go

@@ -0,0 +1,125 @@
+package acceptanceTests
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os/exec"
+	"testing"
+	"time"
+)
+
+func TestTapAndFetch(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	tests := []int{1, 100}
+
+	for _, entriesCount := range tests {
+		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
+			cliPath, cliPathErr := GetCliPath()
+			if cliPathErr != nil {
+				t.Errorf("failed to get cli path, err: %v", cliPathErr)
+				return
+			}
+
+			tapCmdArgs := GetDefaultTapCommandArgs()
+			tapCmd := exec.Command(cliPath, tapCmdArgs...)
+			t.Logf("running command: %v", tapCmd.String())
+
+			t.Cleanup(func() {
+				if err := CleanupCommand(tapCmd); err != nil {
+					t.Logf("failed to cleanup tap command, err: %v", err)
+				}
+			})
+
+			if err := tapCmd.Start(); err != nil {
+				t.Errorf("failed to start tap command, err: %v", err)
+				return
+			}
+
+			time.Sleep(30 * time.Second)
+
+			proxyUrl := "http://localhost:8080/api/v1/namespaces/mizu-tests/services/httpbin/proxy/get"
+			for i := 0; i < entriesCount; i++ {
+				if _, requestErr := ExecuteHttpRequest(proxyUrl); requestErr != nil {
+					t.Errorf("failed to send proxy request, err: %v", requestErr)
+					return
+				}
+			}
+
+			time.Sleep(5 * time.Second)
+			timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+
+			entriesUrl := fmt.Sprintf("http://localhost:8899/mizu/api/entries?limit=%v&operator=lt&timestamp=%v", entriesCount, timestamp)
+			requestResult, requestErr := ExecuteHttpRequest(entriesUrl)
+			if requestErr != nil {
+				t.Errorf("failed to get entries, err: %v", requestErr)
+				return
+			}
+
+			entries, ok := requestResult.([]interface{})
+			if !ok {
+				t.Errorf("invalid entries type")
+				return
+			}
+
+			if len(entries) != entriesCount {
+				t.Errorf("unexpected entries result - Expected: %v, actual: %v", entriesCount, len(entries))
+				return
+			}
+
+			entry, ok :=  entries[0].(map[string]interface{})
+			if !ok {
+				t.Errorf("invalid entry type")
+				return
+			}
+
+			entryUrl := fmt.Sprintf("http://localhost:8899/mizu/api/entries/%v", entry["id"])
+			requestResult, requestErr = ExecuteHttpRequest(entryUrl)
+			if requestErr != nil {
+				t.Errorf("failed to get entry, err: %v", requestErr)
+				return
+			}
+
+			if requestResult == nil {
+				t.Errorf("unexpected nil entry result")
+				return
+			}
+
+			fetchCmdArgs := GetDefaultFetchCommandArgs()
+			fetchCmd := exec.Command(cliPath, fetchCmdArgs...)
+			t.Logf("running command: %v", fetchCmd.String())
+
+			t.Cleanup(func() {
+				if err := CleanupCommand(fetchCmd); err != nil {
+					t.Logf("failed to cleanup fetch command, err: %v", err)
+				}
+			})
+
+			if err := fetchCmd.Start(); err != nil {
+				t.Errorf("failed to start fetch command, err: %v", err)
+				return
+			}
+
+			time.Sleep(5 * time.Second)
+
+			harBytes, readFileErr := ioutil.ReadFile("./unknown_source.har")
+			if readFileErr != nil {
+				t.Errorf("failed to read har file, err: %v", readFileErr)
+				return
+			}
+
+			harEntries, err := GetEntriesFromHarBytes(harBytes)
+			if err != nil {
+				t.Errorf("failed to get entries from har, err: %v", err)
+				return
+			}
+
+			if len(harEntries) != entriesCount {
+				t.Errorf("unexpected har entries result - Expected: %v, actual: %v", entriesCount, len(harEntries))
+				return
+			}
+		})
+	}
+}

acceptanceTests/testsUtils.go

@@ -0,0 +1,113 @@
+package acceptanceTests
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"syscall"
+)
+
+func GetCliPath() (string, error) {
+	dir, filePathErr := os.Getwd()
+	if filePathErr != nil {
+		return "", filePathErr
+	}
+
+	cliPath := path.Join(dir, "../cli/bin/mizu_ci")
+	return cliPath, nil
+}
+
+func GetDefaultCommandArgs() []string {
+	setFlag := "--set"
+	telemetry := "telemetry=false"
+
+	return []string{setFlag, telemetry}
+}
+
+func GetDefaultTapCommandArgs() []string {
+	tapCommand := "tap"
+	setFlag := "--set"
+	namespaces := "tap.namespaces=mizu-tests"
+	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0"
+	imagePullPolicy := "image-pull-policy=Never"
+
+	defaultCmdArgs := GetDefaultCommandArgs()
+
+	return append([]string{tapCommand, setFlag, namespaces, setFlag, agentImage, setFlag, imagePullPolicy}, defaultCmdArgs...)
+}
+
+func GetDefaultFetchCommandArgs() []string {
+	tapCommand := "fetch"
+
+	defaultCmdArgs := GetDefaultCommandArgs()
+
+	return append([]string{tapCommand}, defaultCmdArgs...)
+}
+
+func JsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
+	var result interface{}
+	if parseErr := json.Unmarshal(jsonBytes, &result); parseErr != nil {
+		return nil, parseErr
+	}
+
+	return result, nil
+}
+
+func ExecuteHttpRequest(url string) (interface{}, error) {
+	response, requestErr := http.Get(url)
+	if requestErr != nil {
+		return nil, requestErr
+	} else if response.StatusCode != 200 {
+		return nil, fmt.Errorf("invalid status code %v", response.StatusCode)
+	}
+
+	data, readErr := ioutil.ReadAll(response.Body)
+	if readErr != nil {
+		return nil, readErr
+	}
+
+	return JsonBytesToInterface(data)
+}
+
+func CleanupCommand(cmd *exec.Cmd) error {
+	if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil {
+		return err
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func GetEntriesFromHarBytes(harBytes []byte) ([]interface{}, error){
+	harInterface, convertErr := JsonBytesToInterface(harBytes)
+	if convertErr != nil {
+		return nil, convertErr
+	}
+
+	har, ok := harInterface.(map[string]interface{})
+	if !ok {
+		return nil, errors.New("invalid har type")
+	}
+
+	harLogInterface := har["log"]
+	harLog, ok :=  harLogInterface.(map[string]interface{})
+	if !ok {
+		return nil, errors.New("invalid har log type")
+	}
+
+	harEntriesInterface := harLog["entries"]
+	harEntries, ok := harEntriesInterface.([]interface{})
+	if !ok {
+		return nil, errors.New("invalid har entries type")
+	}
+
+	return harEntries, nil
+}

build-agent-ci.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+GCP_PROJECT=up9-docker-hub
+REPOSITORY=gcr.io/$GCP_PROJECT
+SERVER_NAME=mizu
+GIT_BRANCH=ci
+
+DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
+SEM_VER=${SEM_VER=0.0.0}
+
+DOCKER_TAGGED_BUILD="$DOCKER_REPO:$SEM_VER"
+
+echo "building $DOCKER_TAGGED_BUILD"
+docker build -t ${DOCKER_TAGGED_BUILD} --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .

build-push-featurebranch.sh

@@ -1,12 +1,14 @@
 #!/bin/bash
 set -e
 
-SERVER_NAME=mizu
 GCP_PROJECT=up9-docker-hub
 REPOSITORY=gcr.io/$GCP_PROJECT
+SERVER_NAME=mizu
 GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
-SEM_VER=${SEM_VER=0.0.0}
+
 DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
+SEM_VER=${SEM_VER=0.0.0}
+
 DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
 
 if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
@@ -21,6 +23,6 @@ docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_
 
 for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
 do
-        echo pushing "$DOCKER_TAG"
-        docker push "$DOCKER_TAG"
+  echo pushing "$DOCKER_TAG"
+  docker push "$DOCKER_TAG"
 done

cli/cmd/tapRunner.go

@@ -181,8 +181,10 @@ func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Pro
 		IsNamespaceRestricted:   config.Config.IsNsRestrictedMode(),
 		MizuApiFilteringOptions: mizuApiFilteringOptions,
 		MaxEntriesDBSizeBytes:   config.Config.Tap.MaxEntriesDBSizeBytes(),
+		Resources:               config.Config.Tap.ApiServerResources,
+		ImagePullPolicy:         config.Config.ImagePullPolicy(),
 	}
-	_, err = kubernetesProvider.CreateMizuApiServerPod(ctx, opts, config.Config.Tap.ApiServerResources)
+	_, err = kubernetesProvider.CreateMizuApiServerPod(ctx, opts)
 	if err != nil {
 		return err
 	}
@@ -238,6 +240,7 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
 			serviceAccountName,
 			config.Config.Tap.TapOutgoing(),
 			config.Config.Tap.TapperResources,
+			config.Config.ImagePullPolicy(),
 		); err != nil {
 			return err
 		}

cli/config/configStruct.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/mizu"
+	v1 "k8s.io/api/core/v1"
 )
 
 const (
@@ -16,6 +17,7 @@ type ConfigStruct struct {
 	Version                configStructs.VersionConfig `yaml:"version"`
 	View                   configStructs.ViewConfig    `yaml:"view"`
 	AgentImage             string                      `yaml:"agent-image,omitempty" readonly:""`
+	ImagePullPolicyStr     string                      `yaml:"image-pull-policy" default:"Always"`
 	MizuResourcesNamespace string                      `yaml:"mizu-resources-namespace" default:"mizu"`
 	Telemetry              bool                        `yaml:"telemetry" default:"true"`
 	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`
@@ -26,6 +28,10 @@ func (config *ConfigStruct) SetDefaults() {
 	config.AgentImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", mizu.Branch, mizu.SemVer)
 }
 
+func (config *ConfigStruct) ImagePullPolicy() v1.PullPolicy {
+	return v1.PullPolicy(config.ImagePullPolicyStr)
+}
+
 func (config *ConfigStruct) IsNsRestrictedMode() bool {
 	return config.MizuResourcesNamespace != "mizu" // Notice "mizu" string must match the default MizuResourcesNamespace
 }

cli/kubernetes/provider.go

@@ -142,9 +142,11 @@ type ApiServerOptions struct {
 	IsNamespaceRestricted   bool
 	MizuApiFilteringOptions *shared.TrafficFilteringOptions
 	MaxEntriesDBSizeBytes   int64
+	Resources               configStructs.Resources
+	ImagePullPolicy         core.PullPolicy
 }
 
-func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiServerOptions, resources configStructs.Resources) (*core.Pod, error) {
+func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiServerOptions) (*core.Pod, error) {
 	marshaledFilteringOptions, err := json.Marshal(opts.MizuApiFilteringOptions)
 	if err != nil {
 		return nil, err
@@ -154,19 +156,19 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS
 	configMapOptional := true
 	configMapVolumeName.Optional = &configMapOptional
 
-	cpuLimit, err := resource.ParseQuantity(resources.CpuLimit)
+	cpuLimit, err := resource.ParseQuantity(opts.Resources.CpuLimit)
 	if err != nil {
 		return nil, errors.New(fmt.Sprintf("invalid cpu limit for %s container", opts.PodName))
 	}
-	memLimit, err := resource.ParseQuantity(resources.MemoryLimit)
+	memLimit, err := resource.ParseQuantity(opts.Resources.MemoryLimit)
 	if err != nil {
 		return nil, errors.New(fmt.Sprintf("invalid memory limit for %s container", opts.PodName))
 	}
-	cpuRequests, err := resource.ParseQuantity(resources.CpuRequests)
+	cpuRequests, err := resource.ParseQuantity(opts.Resources.CpuRequests)
 	if err != nil {
 		return nil, errors.New(fmt.Sprintf("invalid cpu request for %s container", opts.PodName))
 	}
-	memRequests, err := resource.ParseQuantity(resources.MemoryRequests)
+	memRequests, err := resource.ParseQuantity(opts.Resources.MemoryRequests)
 	if err != nil {
 		return nil, errors.New(fmt.Sprintf("invalid memory request for %s container", opts.PodName))
 	}
@@ -187,7 +189,7 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS
 				{
 					Name:            opts.PodName,
 					Image:           opts.PodImage,
-					ImagePullPolicy: core.PullAlways,
+					ImagePullPolicy: opts.ImagePullPolicy,
 					VolumeMounts: []core.VolumeMount{
 						{
 							Name:      mizu.ConfigMapName,
@@ -563,7 +565,7 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
 	return nil
 }
 
-func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool, resources configStructs.Resources) error {
+func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool, resources configStructs.Resources, imagePullPolicy core.PullPolicy) error {
 	logger.Log.Debugf("Applying %d tapper deamonsets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
 
 	if len(nodeToTappedPodIPMap) == 0 {
@@ -588,7 +590,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	agentContainer := applyconfcore.Container()
 	agentContainer.WithName(tapperPodName)
 	agentContainer.WithImage(podImage)
-	agentContainer.WithImagePullPolicy(core.PullAlways)
+	agentContainer.WithImagePullPolicy(imagePullPolicy)
 	agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(true))
 	agentContainer.WithCommand(mizuCmd...)
 	agentContainer.WithEnv(