Change tests to use github repo (#1211)

remove goole related stuff

.github/workflows/acceptance_tests.yml

@@ -52,15 +52,3 @@ jobs:
 
       - name: Test
         run: make acceptance-test
-
-      - name: Slack notification on failure
-        uses: ravsamhq/notify-slack-action@v1
-        if: always()
-        with:
-          status: ${{ job.status }}
-          notification_title: 'Mizu {workflow} has {status_message}'
-          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit <{commit_url}|{commit_sha}> by ${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}> ```${{ github.event.head_commit.message }}```'
-          footer: 'Linked Repo <{repo_url}|{repo}>'
-          notify_when: 'failure'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/release.yml

@@ -85,94 +85,6 @@ jobs:
             GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
             COMMIT_HASH=${{ github.sha }}
 
-  gcp-registry:
-    name: Push Docker image to GCR
-    runs-on: ubuntu-latest
-
-    strategy:
-      max-parallel: 2
-      fail-fast: false
-      matrix:
-        target:
-          - amd64
-          - arm64v8
-
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v2
-
-      - id: 'auth'
-        uses: 'google-github-actions/auth@v0'
-        with:
-          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
-
-      - name: 'Set up Cloud SDK'
-        uses: 'google-github-actions/setup-gcloud@v0'
-
-      - name: Determine versioning strategy
-        uses: haya14busa/action-cond@v1
-        id: condval
-        with:
-          cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "stable"
-          if_false: "dev"
-
-      - name: Auto Increment Ver Action
-        uses: docker://igorgov/auto-inc-ver:v2.0.0
-        id: versioning
-        with:
-          mode: ${{ steps.condval.outputs.value }}
-          suffix: 'dev'
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Get version parameters
-        shell: bash
-        run: |
-          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
-          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: version_parameters
-
-      - name: Get base image name
-        shell: bash
-        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
-        id: base_image_step
-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-            ${{ steps.base_image_step.outputs.image }}
-          tags: |
-            type=raw,${{ steps.versioning.outputs.version }}
-            type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
-            type=raw,value=dev-latest,enable=${{ steps.condval.outputs.value == 'dev' }}
-          flavor: |
-            latest=auto
-            prefix=
-            suffix=-${{ matrix.target }},onlatest=true
-
-      - name: Login to GCR
-        uses: docker/login-action@v1
-        with:
-          registry: gcr.io
-          username: _json_key
-          password: ${{ secrets.GCR_JSON_KEY }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            TARGETARCH=${{ matrix.target }}
-            VER=${{ steps.versioning.outputs.version }}
-            BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
-            GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
-            COMMIT_HASH=${{ github.sha }}
-
   docker-manifest:
     name: Create and Push a Docker Manifest
     runs-on: ubuntu-latest
@@ -232,7 +144,7 @@ jobs:
   cli:
     name: Build the CLI and publish
     runs-on: ubuntu-latest
-    needs: [docker-manifest, gcp-registry]
+    needs: [docker-manifest]
     steps:
       - name: Set up Go 1.17
         uses: actions/setup-go@v2
@@ -290,15 +202,3 @@ jobs:
           tag: ${{ steps.versioning.outputs.version }}
           prerelease: ${{ github.ref != 'refs/heads/main' }}
           bodyFile: 'cli/bin/README.md'
-      
-      - name: Slack notification on failure
-        uses: ravsamhq/notify-slack-action@v1
-        if: always()
-        with:
-          status: ${{ job.status }}
-          notification_title: 'Mizu enterprise {workflow} has {status_message}'
-          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit <{commit_url}|{commit_sha}> by ${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}> ```${{ github.event.head_commit.message }}```'
-          footer: 'Linked Repo <{repo_url}|{repo}>'
-          notify_when: 'failure'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

tap/extensions/amqp/Makefile

@@ -8,9 +8,7 @@ test-update: test-pull-bin
 	@MIZU_TEST=1 TEST_UPDATE=1 go test -v ./... -coverpkg=./... -coverprofile=coverage.out -covermode=atomic
 
 test-pull-bin:
-	@mkdir -p bin
+	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || ../get-folder-of-tests.sh bin/amqp bin
-	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp gs://static.up9.io/mizu/test-pcap/bin/amqp/\*.bin bin
 
 test-pull-expect:
-	@mkdir -p expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || ../get-folder-of-tests.sh expect/amqp expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect16/amqp/\* expect

tap/extensions/get-folder-of-tests.sh

@@ -0,0 +1,24 @@
+mizu_test_files_repo=https://github.com/up9inc/mizu-tests-data
+mizu_test_files_tmp_folder="mizu-test-files-tmp"
+
+requested_folder=$1
+destination_folder_name=$2
+echo "Going to download folder (${requested_folder}) from repo (${mizu_test_files_repo}) and save it to local folder (${destination_folder_name})"
+
+echo "Cloning repo to tmp folder (${mizu_test_files_tmp_folder})"
+git clone ${mizu_test_files_repo} ${mizu_test_files_tmp_folder} --no-checkout --depth 1 --filter=blob:none --sparse
+cd ${mizu_test_files_tmp_folder}
+
+echo "Adding sparse checkout folder"
+git sparse-checkout add ${requested_folder}
+
+
+echo "Checkout"
+git checkout 
+
+echo "Moving folder to the destination location"
+mv ${requested_folder} ../${destination_folder_name}
+
+cd ..
+echo "Removing the tmp folder"
+rm -rf ${mizu_test_files_tmp_folder}

tap/extensions/http/Makefile

@@ -8,9 +8,7 @@ test-update: test-pull-bin
 	@MIZU_TEST=1 TEST_UPDATE=1 go test -v ./... -coverpkg=./... -coverprofile=coverage.out -covermode=atomic
 
 test-pull-bin:
-	@mkdir -p bin
+	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || ../get-folder-of-tests.sh bin/http bin
-	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp gs://static.up9.io/mizu/test-pcap/bin/http/\*.bin bin
 
 test-pull-expect:
-	@mkdir -p expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || ../get-folder-of-tests.sh expect/http expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect16/http/\* expect

tap/extensions/kafka/Makefile

@@ -8,9 +8,7 @@ test-update: test-pull-bin
 	@MIZU_TEST=1 TEST_UPDATE=1 go test -v ./... -coverpkg=./... -coverprofile=coverage.out -covermode=atomic
 
 test-pull-bin:
-	@mkdir -p bin
+	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || ../get-folder-of-tests.sh bin/kafka bin
-	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp gs://static.up9.io/mizu/test-pcap/bin/kafka/\*.bin bin
 
 test-pull-expect:
-	@mkdir -p expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || ../get-folder-of-tests.sh expect/kafka expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect15/kafka/\* expect

tap/extensions/redis/Makefile

@@ -8,9 +8,7 @@ test-update: test-pull-bin
 	@MIZU_TEST=1 TEST_UPDATE=1 go test -v ./... -coverpkg=./... -coverprofile=coverage.out -covermode=atomic
 
 test-pull-bin:
-	@mkdir -p bin
+	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || ../get-folder-of-tests.sh bin/redis bin
-	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp gs://static.up9.io/mizu/test-pcap/bin/redis/\*.bin bin
 
 test-pull-expect:
-	@mkdir -p expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || ../get-folder-of-tests.sh expect/redis expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect15/redis/\* expect