From db1f4458c53430911a989c957c5d35ab230cefe0 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Wed, 18 Aug 2021 10:22:45 +0300 Subject: [PATCH] Introducing acceptance test (#222) --- .github/workflows/acceptance_tests.yml | 26 ++++ .../{validations.yaml => validations.yml} | 0 Makefile | 15 ++- acceptanceTests/Makefile | 2 + acceptanceTests/go.mod | 3 + acceptanceTests/setup.sh | 48 +++++++ acceptanceTests/tap_test.go | 125 ++++++++++++++++++ acceptanceTests/testsUtils.go | 113 ++++++++++++++++ build-agent-ci.sh | 15 +++ build-push-featurebranch.sh | 10 +- cli/cmd/tapRunner.go | 5 +- cli/config/configStruct.go | 6 + cli/kubernetes/provider.go | 18 +-- 13 files changed, 370 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/acceptance_tests.yml rename .github/workflows/{validations.yaml => validations.yml} (100%) create mode 100644 acceptanceTests/Makefile create mode 100644 acceptanceTests/go.mod create mode 100644 acceptanceTests/setup.sh create mode 100644 acceptanceTests/tap_test.go create mode 100644 acceptanceTests/testsUtils.go create mode 100755 build-agent-ci.sh diff --git a/.github/workflows/acceptance_tests.yml b/.github/workflows/acceptance_tests.yml new file mode 100644 index 000000000..66d349b91 --- /dev/null +++ b/.github/workflows/acceptance_tests.yml @@ -0,0 +1,26 @@ +name: acceptance tests +on: + pull_request: + branches: + - 'main' + push: + branches: + - 'develop' +jobs: + run-acceptance-tests: + name: Run acceptance tests + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.16 + uses: actions/setup-go@v2 + with: + go-version: '^1.16' + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Setup acceptance test + run: source ./acceptanceTests/setup.sh + + - name: Test + run: make acceptance-test diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yml similarity index 100% rename from .github/workflows/validations.yaml rename to .github/workflows/validations.yml diff --git a/Makefile b/Makefile index 6a83d79ff..2807c6283 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,9 @@ ui: ## Build UI. cli: ## Build CLI. @echo "building cli"; cd cli && $(MAKE) build +build-cli-ci: ## Build CLI for CI. + @echo "building cli for ci"; cd cli && $(MAKE) build GIT_BRANCH=ci SUFFIX=ci + agent: ## Build agent. @(echo "building mizu agent .." ) @(cd agent; go build -o build/mizuagent main.go) @@ -42,6 +45,10 @@ push-docker: ## Build and publish agent docker image. @echo "publishing Docker image .. " ./build-push-featurebranch.sh +build-docker-ci: ## Build agent docker image for CI. + @echo "building docker image for ci" + ./build-agent-ci.sh + push-cli: ## Build and publish CLI. @echo "publishing CLI .. " @cd cli; $(MAKE) build-all @@ -50,7 +57,6 @@ push-cli: ## Build and publish CLI. gsutil cp -r ./cli/bin/* gs://${BUCKET_PATH}/ gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\* - clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts. clean-ui: ## Clean UI. @@ -65,8 +71,11 @@ clean-cli: ## Clean CLI. clean-docker: @(echo "DOCKER cleanup - NOT IMPLEMENTED YET " ) -test-cli: ## Run tests. +test-cli: @echo "running cli tests"; cd cli && $(MAKE) test -test-agent: ## Run tests. +test-agent: @echo "running agent tests"; cd agent && $(MAKE) test + +acceptance-test: + @echo "running acceptance tests"; cd acceptanceTests && $(MAKE) test diff --git a/acceptanceTests/Makefile b/acceptanceTests/Makefile new file mode 100644 index 000000000..8a142a160 --- /dev/null +++ b/acceptanceTests/Makefile @@ -0,0 +1,2 @@ +test: ## Run acceptance tests. + @go test ./... diff --git a/acceptanceTests/go.mod b/acceptanceTests/go.mod new file mode 100644 index 000000000..bc529ac9d --- /dev/null +++ b/acceptanceTests/go.mod @@ -0,0 +1,3 @@ +module github.com/up9inc/mizu/tests + +go 1.16 diff --git a/acceptanceTests/setup.sh b/acceptanceTests/setup.sh new file mode 100644 index 000000000..0ed9dbce1 --- /dev/null +++ b/acceptanceTests/setup.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +PREFIX=$HOME/local/bin +VERSION=v1.22.0 + +echo "Attempting to install minikube and assorted tools to $PREFIX" + +if ! [ -x "$(command -v kubectl)" ]; then + echo "Installing kubectl version $VERSION" + curl -LO "https://storage.googleapis.com/kubernetes-release/release/$VERSION/bin/linux/amd64/kubectl" + chmod +x kubectl + mv kubectl "$PREFIX" +else + echo "kubetcl is already installed" +fi + +if ! [ -x "$(command -v minikube)" ]; then + echo "Installing minikube version $VERSION" + curl -Lo minikube https://storage.googleapis.com/minikube/releases/$VERSION/minikube-linux-amd64 + chmod +x minikube + mv minikube "$PREFIX" +else + echo "minikube is already installed" +fi + +echo "Starting minikube..." +minikube start + +echo "Creating mizu tests namespace" +kubectl create namespace mizu-tests + +echo "Creating httpbin deployment" +kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests + +echo "Creating httpbin service" +kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests + +echo "Starting proxy" +kubectl proxy --port=8080 & + +echo "Setting minikube docker env" +eval $(minikube docker-env) + +echo "Build agent image" +make build-docker-ci + +echo "Build cli" +make build-cli-ci diff --git a/acceptanceTests/tap_test.go b/acceptanceTests/tap_test.go new file mode 100644 index 000000000..3c359a0d0 --- /dev/null +++ b/acceptanceTests/tap_test.go @@ -0,0 +1,125 @@ +package acceptanceTests + +import ( + "fmt" + "io/ioutil" + "os/exec" + "testing" + "time" +) + +func TestTapAndFetch(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + tests := []int{1, 100} + + for _, entriesCount := range tests { + t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) { + cliPath, cliPathErr := GetCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + tapCmdArgs := GetDefaultTapCommandArgs() + tapCmd := exec.Command(cliPath, tapCmdArgs...) + t.Logf("running command: %v", tapCmd.String()) + + t.Cleanup(func() { + if err := CleanupCommand(tapCmd); err != nil { + t.Logf("failed to cleanup tap command, err: %v", err) + } + }) + + if err := tapCmd.Start(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + time.Sleep(30 * time.Second) + + proxyUrl := "http://localhost:8080/api/v1/namespaces/mizu-tests/services/httpbin/proxy/get" + for i := 0; i < entriesCount; i++ { + if _, requestErr := ExecuteHttpRequest(proxyUrl); requestErr != nil { + t.Errorf("failed to send proxy request, err: %v", requestErr) + return + } + } + + time.Sleep(5 * time.Second) + timestamp := time.Now().UnixNano() / int64(time.Millisecond) + + entriesUrl := fmt.Sprintf("http://localhost:8899/mizu/api/entries?limit=%v&operator=lt×tamp=%v", entriesCount, timestamp) + requestResult, requestErr := ExecuteHttpRequest(entriesUrl) + if requestErr != nil { + t.Errorf("failed to get entries, err: %v", requestErr) + return + } + + entries, ok := requestResult.([]interface{}) + if !ok { + t.Errorf("invalid entries type") + return + } + + if len(entries) != entriesCount { + t.Errorf("unexpected entries result - Expected: %v, actual: %v", entriesCount, len(entries)) + return + } + + entry, ok := entries[0].(map[string]interface{}) + if !ok { + t.Errorf("invalid entry type") + return + } + + entryUrl := fmt.Sprintf("http://localhost:8899/mizu/api/entries/%v", entry["id"]) + requestResult, requestErr = ExecuteHttpRequest(entryUrl) + if requestErr != nil { + t.Errorf("failed to get entry, err: %v", requestErr) + return + } + + if requestResult == nil { + t.Errorf("unexpected nil entry result") + return + } + + fetchCmdArgs := GetDefaultFetchCommandArgs() + fetchCmd := exec.Command(cliPath, fetchCmdArgs...) + t.Logf("running command: %v", fetchCmd.String()) + + t.Cleanup(func() { + if err := CleanupCommand(fetchCmd); err != nil { + t.Logf("failed to cleanup fetch command, err: %v", err) + } + }) + + if err := fetchCmd.Start(); err != nil { + t.Errorf("failed to start fetch command, err: %v", err) + return + } + + time.Sleep(5 * time.Second) + + harBytes, readFileErr := ioutil.ReadFile("./unknown_source.har") + if readFileErr != nil { + t.Errorf("failed to read har file, err: %v", readFileErr) + return + } + + harEntries, err := GetEntriesFromHarBytes(harBytes) + if err != nil { + t.Errorf("failed to get entries from har, err: %v", err) + return + } + + if len(harEntries) != entriesCount { + t.Errorf("unexpected har entries result - Expected: %v, actual: %v", entriesCount, len(harEntries)) + return + } + }) + } +} diff --git a/acceptanceTests/testsUtils.go b/acceptanceTests/testsUtils.go new file mode 100644 index 000000000..123151ac3 --- /dev/null +++ b/acceptanceTests/testsUtils.go @@ -0,0 +1,113 @@ +package acceptanceTests + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path" + "syscall" +) + +func GetCliPath() (string, error) { + dir, filePathErr := os.Getwd() + if filePathErr != nil { + return "", filePathErr + } + + cliPath := path.Join(dir, "../cli/bin/mizu_ci") + return cliPath, nil +} + +func GetDefaultCommandArgs() []string { + setFlag := "--set" + telemetry := "telemetry=false" + + return []string{setFlag, telemetry} +} + +func GetDefaultTapCommandArgs() []string { + tapCommand := "tap" + setFlag := "--set" + namespaces := "tap.namespaces=mizu-tests" + agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0" + imagePullPolicy := "image-pull-policy=Never" + + defaultCmdArgs := GetDefaultCommandArgs() + + return append([]string{tapCommand, setFlag, namespaces, setFlag, agentImage, setFlag, imagePullPolicy}, defaultCmdArgs...) +} + +func GetDefaultFetchCommandArgs() []string { + tapCommand := "fetch" + + defaultCmdArgs := GetDefaultCommandArgs() + + return append([]string{tapCommand}, defaultCmdArgs...) +} + +func JsonBytesToInterface(jsonBytes []byte) (interface{}, error) { + var result interface{} + if parseErr := json.Unmarshal(jsonBytes, &result); parseErr != nil { + return nil, parseErr + } + + return result, nil +} + +func ExecuteHttpRequest(url string) (interface{}, error) { + response, requestErr := http.Get(url) + if requestErr != nil { + return nil, requestErr + } else if response.StatusCode != 200 { + return nil, fmt.Errorf("invalid status code %v", response.StatusCode) + } + + data, readErr := ioutil.ReadAll(response.Body) + if readErr != nil { + return nil, readErr + } + + return JsonBytesToInterface(data) +} + +func CleanupCommand(cmd *exec.Cmd) error { + if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil { + return err + } + + if err := cmd.Wait(); err != nil { + return err + } + + return nil +} + +func GetEntriesFromHarBytes(harBytes []byte) ([]interface{}, error){ + harInterface, convertErr := JsonBytesToInterface(harBytes) + if convertErr != nil { + return nil, convertErr + } + + har, ok := harInterface.(map[string]interface{}) + if !ok { + return nil, errors.New("invalid har type") + } + + harLogInterface := har["log"] + harLog, ok := harLogInterface.(map[string]interface{}) + if !ok { + return nil, errors.New("invalid har log type") + } + + harEntriesInterface := harLog["entries"] + harEntries, ok := harEntriesInterface.([]interface{}) + if !ok { + return nil, errors.New("invalid har entries type") + } + + return harEntries, nil +} diff --git a/build-agent-ci.sh b/build-agent-ci.sh new file mode 100755 index 000000000..430177a9c --- /dev/null +++ b/build-agent-ci.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +GCP_PROJECT=up9-docker-hub +REPOSITORY=gcr.io/$GCP_PROJECT +SERVER_NAME=mizu +GIT_BRANCH=ci + +DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH +SEM_VER=${SEM_VER=0.0.0} + +DOCKER_TAGGED_BUILD="$DOCKER_REPO:$SEM_VER" + +echo "building $DOCKER_TAGGED_BUILD" +docker build -t ${DOCKER_TAGGED_BUILD} --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} . diff --git a/build-push-featurebranch.sh b/build-push-featurebranch.sh index 0441c3960..4e040ac69 100755 --- a/build-push-featurebranch.sh +++ b/build-push-featurebranch.sh @@ -1,12 +1,14 @@ #!/bin/bash set -e -SERVER_NAME=mizu GCP_PROJECT=up9-docker-hub REPOSITORY=gcr.io/$GCP_PROJECT +SERVER_NAME=mizu GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]') -SEM_VER=${SEM_VER=0.0.0} + DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH +SEM_VER=${SEM_VER=0.0.0} + DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER") if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ] @@ -21,6 +23,6 @@ docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_ for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}" do - echo pushing "$DOCKER_TAG" - docker push "$DOCKER_TAG" + echo pushing "$DOCKER_TAG" + docker push "$DOCKER_TAG" done diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index c3c8713a7..92d42d19a 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -181,8 +181,10 @@ func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Pro IsNamespaceRestricted: config.Config.IsNsRestrictedMode(), MizuApiFilteringOptions: mizuApiFilteringOptions, MaxEntriesDBSizeBytes: config.Config.Tap.MaxEntriesDBSizeBytes(), + Resources: config.Config.Tap.ApiServerResources, + ImagePullPolicy: config.Config.ImagePullPolicy(), } - _, err = kubernetesProvider.CreateMizuApiServerPod(ctx, opts, config.Config.Tap.ApiServerResources) + _, err = kubernetesProvider.CreateMizuApiServerPod(ctx, opts) if err != nil { return err } @@ -238,6 +240,7 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi serviceAccountName, config.Config.Tap.TapOutgoing(), config.Config.Tap.TapperResources, + config.Config.ImagePullPolicy(), ); err != nil { return err } diff --git a/cli/config/configStruct.go b/cli/config/configStruct.go index db7aad9b7..431e5d67e 100644 --- a/cli/config/configStruct.go +++ b/cli/config/configStruct.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/up9inc/mizu/cli/config/configStructs" "github.com/up9inc/mizu/cli/mizu" + v1 "k8s.io/api/core/v1" ) const ( @@ -16,6 +17,7 @@ type ConfigStruct struct { Version configStructs.VersionConfig `yaml:"version"` View configStructs.ViewConfig `yaml:"view"` AgentImage string `yaml:"agent-image,omitempty" readonly:""` + ImagePullPolicyStr string `yaml:"image-pull-policy" default:"Always"` MizuResourcesNamespace string `yaml:"mizu-resources-namespace" default:"mizu"` Telemetry bool `yaml:"telemetry" default:"true"` DumpLogs bool `yaml:"dump-logs" default:"false"` @@ -26,6 +28,10 @@ func (config *ConfigStruct) SetDefaults() { config.AgentImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", mizu.Branch, mizu.SemVer) } +func (config *ConfigStruct) ImagePullPolicy() v1.PullPolicy { + return v1.PullPolicy(config.ImagePullPolicyStr) +} + func (config *ConfigStruct) IsNsRestrictedMode() bool { return config.MizuResourcesNamespace != "mizu" // Notice "mizu" string must match the default MizuResourcesNamespace } diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index 5559b0685..81ef70a45 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -142,9 +142,11 @@ type ApiServerOptions struct { IsNamespaceRestricted bool MizuApiFilteringOptions *shared.TrafficFilteringOptions MaxEntriesDBSizeBytes int64 + Resources configStructs.Resources + ImagePullPolicy core.PullPolicy } -func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiServerOptions, resources configStructs.Resources) (*core.Pod, error) { +func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiServerOptions) (*core.Pod, error) { marshaledFilteringOptions, err := json.Marshal(opts.MizuApiFilteringOptions) if err != nil { return nil, err @@ -154,19 +156,19 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS configMapOptional := true configMapVolumeName.Optional = &configMapOptional - cpuLimit, err := resource.ParseQuantity(resources.CpuLimit) + cpuLimit, err := resource.ParseQuantity(opts.Resources.CpuLimit) if err != nil { return nil, errors.New(fmt.Sprintf("invalid cpu limit for %s container", opts.PodName)) } - memLimit, err := resource.ParseQuantity(resources.MemoryLimit) + memLimit, err := resource.ParseQuantity(opts.Resources.MemoryLimit) if err != nil { return nil, errors.New(fmt.Sprintf("invalid memory limit for %s container", opts.PodName)) } - cpuRequests, err := resource.ParseQuantity(resources.CpuRequests) + cpuRequests, err := resource.ParseQuantity(opts.Resources.CpuRequests) if err != nil { return nil, errors.New(fmt.Sprintf("invalid cpu request for %s container", opts.PodName)) } - memRequests, err := resource.ParseQuantity(resources.MemoryRequests) + memRequests, err := resource.ParseQuantity(opts.Resources.MemoryRequests) if err != nil { return nil, errors.New(fmt.Sprintf("invalid memory request for %s container", opts.PodName)) } @@ -187,7 +189,7 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS { Name: opts.PodName, Image: opts.PodImage, - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: opts.ImagePullPolicy, VolumeMounts: []core.VolumeMount{ { Name: mizu.ConfigMapName, @@ -563,7 +565,7 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, return nil } -func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool, resources configStructs.Resources) error { +func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool, resources configStructs.Resources, imagePullPolicy core.PullPolicy) error { logger.Log.Debugf("Applying %d tapper deamonsets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName) if len(nodeToTappedPodIPMap) == 0 { @@ -588,7 +590,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac agentContainer := applyconfcore.Container() agentContainer.WithName(tapperPodName) agentContainer.WithImage(podImage) - agentContainer.WithImagePullPolicy(core.PullAlways) + agentContainer.WithImagePullPolicy(imagePullPolicy) agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(true)) agentContainer.WithCommand(mizuCmd...) agentContainer.WithEnv(