Add ARM64 and cross-compilation support to the agent image (#659)

* modified Dockerfile to work for both amd64 (Intel) and arm64 (M1)

* added changelog

* Update `Dockerfile` to have `ARCH` build argument

* Remove `docs/CHANGES.md`

* Upgrade the Basenine version from `v0.3.0` to `v0.4.6`

* Update `publish.yml` to have `ARCH` build argument

* Switch `BasenineImageRepo` to Docker Hub

* Have separate build arguments for `ARCH` and `GOARCH`

* Upgrade the Basenine version from `v0.4.6` to `v0.4.10`

* Oops forgot to update the 10th duplicated shell script

* Fix the oopsie and reduce duplications

* Fix `Dockerfile`

* Fix the incompatibility issue between Go plugins and gold linker in Alpine inside `Dockerfile`

* Fix `asm: xxhash_amd64.s:120: when dynamic linking, R15 is clobbered by a global variable access` error

* Update `Dockerfile` to have cross-compilation on an AMD64 machine

Also revert changes in the shell scripts

* Delete `debug.Dockerfile`

* Create a custom base (`debian:buster-slim` based) image for the shipped image

* Replace `mertyildiran/debian-pcap` with `up9inc/debian-pcap`

* Upgrade Basenine version to `v0.4.12`

* Use `debian:stable-slim` as the base

* Fix the indentation in the `Dockerfile`

* Update `publish.yml`

* Enable `publish.yml` for `feature/multiarch_build` branch

* Tag correctly and set `ARCH` Docker argument

* Remove the lines that are forgotten to be removed from the shell scripts

* Add `MizuAgentImageRepo` constant and use it as default `AgentImage` value

* Bring back `Set up Cloud SDK` step to `Build the CLI and publish` job

* Build ARM64 CLI for Linux as well

* Revert "Enable `publish.yml` for `feature/multiarch_build` branch"

This reverts commit d30be4c1f0.

* Revert Go 1.17 upgrade

* Remove `build_extensions_debug.sh` as well

* Make the `Dockerfile` to compile the agent statically

* Statically link the protocol extensions

* Fix `Dockerfile`

* Bring back `-s -w` flags

* Verify the signatures of the downloads in `dockcross/linux-arm64-musl`

* Revert modifications in some shell scripts

* Make the `BUILDARCH` and `TARGETARCH` separation in the `Dockerfile`

* Separate cross-compilation builder image into a separate repo named `up9inc/linux-arm64-musl-go-libpcap`

* Fill the shell script and specify the tag for `dockcross/linux-arm64-musl`

* Remove the unnecessary dependencies from `builder-native-base`

* Improve the comments in the `Dockerfile`

* Upgrade Basenine version to `v0.4.13`

* Fix `Dockerfile`

* Revert "Revert "Enable `publish.yml` for `feature/multiarch_build` branch""

This reverts commit 303e466bdc.

* Revert "Revert "Revert "Enable `publish.yml` for `feature/multiarch_build` branch"""

This reverts commit 0fe252bbdb.

* Remove `push-docker-debug` from the `Makefile`

* Rename `publish.yml` to `release.yml`

Co-authored-by: Alex Haiut <alex@up9.com>

.github/workflows/publish.yml

@@ -1,95 +0,0 @@
-name: publish
-
-on:
-  push:
-    branches:
-      - 'develop'
-      - 'main'
-
-concurrency:
-  group: mizu-publish-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  docker:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Go 1.16
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.16'
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@master
-        with:
-          service_account_key: ${{ secrets.GCR_JSON_KEY }}
-          export_default_credentials: true
-      - uses: haya14busa/action-cond@v1
-        id: condval
-        with:
-          cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "minor"
-          if_false: "patch"
-      - name: Auto Increment Semver Action
-        uses: MCKanpolat/auto-semver-action@1.0.5
-        id: versioning
-        with:
-          releaseType: ${{ steps.condval.outputs.value }}
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Get version parameters
-        shell: bash
-        run: |
-          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
-          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: version_parameters
-      - name: Get base image name
-        shell: bash
-        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
-        id: base_image_step
-      - name: Docker meta
-        id: meta
-        uses: crazy-max/ghaction-docker-meta@v2
-        with:
-          images: |
-            ${{ steps.base_image_step.outputs.image }}
-            up9inc/mizu
-          tags: |
-            type=raw,${{ steps.versioning.outputs.version }}
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_PASS }}
-      - name: Login to GCR
-        uses: docker/login-action@v1
-        with:
-          registry: gcr.io
-          username: _json_key
-          password: ${{ secrets.GCR_JSON_KEY }}
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            SEM_VER=${{ steps.versioning.outputs.version }}
-            BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
-            GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
-            COMMIT_HASH=${{ github.sha }}
-      - name: Build and Push CLI
-        run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
-      - shell: bash
-        run: |
-          echo '${{ steps.versioning.outputs.version }}' >> cli/bin/version.txt
-      - name: publish
-        uses: ncipollo/release-action@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          artifacts: "cli/bin/*"
-          commit: ${{ steps.version_parameters.outputs.branch }}
-          tag: ${{ steps.versioning.outputs.version }}
-          prerelease: ${{ github.ref != 'refs/heads/main' }}
-          bodyFile: 'cli/bin/README.md'

.github/workflows/release.yml

@@ -0,0 +1,278 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - 'develop'
+      - 'main'
+
+concurrency:
+  group: mizu-publish-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  docker-registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+
+    strategy:
+      max-parallel: 2
+      fail-fast: false
+      matrix:
+        target:
+          - amd64
+          - arm64v8
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Determine versioning strategy
+        uses: haya14busa/action-cond@v1
+        id: condval
+        with:
+          cond: ${{ github.ref == 'refs/heads/main' }}
+          if_true: "minor"
+          if_false: "patch"
+
+      - name: Auto increment SemVer action
+        uses: MCKanpolat/auto-semver-action@1.0.5
+        id: versioning
+        with:
+          releaseType: ${{ steps.condval.outputs.value }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get version parameters
+        shell: bash
+        run: |
+          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: version_parameters
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            up9inc/mizu
+          tags: |
+            type=raw,${{ steps.versioning.outputs.version }}
+          flavor: |
+            latest=auto
+            prefix=
+            suffix=-${{ matrix.target }},onlatest=true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            TARGETARCH=${{ matrix.target }}
+            SEM_VER=${{ steps.versioning.outputs.version }}
+            BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
+            GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
+            COMMIT_HASH=${{ github.sha }}
+
+  gcp-registry:
+    name: Push Docker image to GCR
+    runs-on: ubuntu-latest
+
+    strategy:
+      max-parallel: 2
+      fail-fast: false
+      matrix:
+        target:
+          - amd64
+          - arm64v8
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@master
+        with:
+          service_account_key: ${{ secrets.GCR_JSON_KEY }}
+          export_default_credentials: true
+
+      - name: Determine versioning strategy
+        uses: haya14busa/action-cond@v1
+        id: condval
+        with:
+          cond: ${{ github.ref == 'refs/heads/main' }}
+          if_true: "minor"
+          if_false: "patch"
+
+      - name: Auto increment SemVer action
+        uses: MCKanpolat/auto-semver-action@1.0.5
+        id: versioning
+        with:
+          releaseType: ${{ steps.condval.outputs.value }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get version parameters
+        shell: bash
+        run: |
+          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: version_parameters
+
+      - name: Get base image name
+        shell: bash
+        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
+        id: base_image_step
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ${{ steps.base_image_step.outputs.image }}
+          tags: |
+            type=raw,${{ steps.versioning.outputs.version }}
+          flavor: |
+            latest=auto
+            prefix=
+            suffix=-${{ matrix.target }},onlatest=true
+
+      - name: Login to GCR
+        uses: docker/login-action@v1
+        with:
+          registry: gcr.io
+          username: _json_key
+          password: ${{ secrets.GCR_JSON_KEY }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            TARGETARCH=${{ matrix.target }}
+            SEM_VER=${{ steps.versioning.outputs.version }}
+            BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
+            GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
+            COMMIT_HASH=${{ github.sha }}
+
+  docker-manifest:
+    name: Create and Push a Docker Manifest
+    runs-on: ubuntu-latest
+    needs: [docker-registry]
+    steps:
+      - name: Determine versioning strategy
+        uses: haya14busa/action-cond@v1
+        id: condval
+        with:
+          cond: ${{ github.ref == 'refs/heads/main' }}
+          if_true: "minor"
+          if_false: "patch"
+
+      - name: Auto increment SemVer action
+        uses: MCKanpolat/auto-semver-action@1.0.5
+        id: versioning
+        with:
+          releaseType: ${{ steps.condval.outputs.value }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get version parameters
+        shell: bash
+        run: |
+          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: version_parameters
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            up9inc/mizu
+          tags: |
+            type=raw,${{ steps.versioning.outputs.version }}
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+
+      - name: Create manifest
+        run: |
+          while IFS= read -r line; do
+            docker manifest create $line --amend $line-amd64 --amend $line-arm64v8
+          done <<< "${{ steps.meta.outputs.tags }}"
+
+      - name: Push manifest
+        run: |
+          while IFS= read -r line; do
+            docker manifest push $line
+          done <<< "${{ steps.meta.outputs.tags }}"
+
+  cli:
+    name: Build the CLI and publish
+    runs-on: ubuntu-latest
+    needs: [docker-manifest, gcp-registry]
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
+
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@master
+        with:
+          service_account_key: ${{ secrets.GCR_JSON_KEY }}
+          export_default_credentials: true
+
+      - uses: haya14busa/action-cond@v1
+        id: condval
+        with:
+          cond: ${{ github.ref == 'refs/heads/main' }}
+          if_true: "minor"
+          if_false: "patch"
+
+      - name: Auto Increment Semver Action
+        uses: MCKanpolat/auto-semver-action@1.0.5
+        id: versioning
+        with:
+          releaseType: ${{ steps.condval.outputs.value }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get version parameters
+        shell: bash
+        run: |
+          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: version_parameters
+
+      - name: Build and Push CLI
+        run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
+
+      - name: Log the version into a .txt file
+        shell: bash
+        run: |
+          echo '${{ steps.versioning.outputs.version }}' >> cli/bin/version.txt
+
+      - name: Release
+        uses: ncipollo/release-action@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          artifacts: "cli/bin/*"
+          commit: ${{ steps.version_parameters.outputs.branch }}
+          tag: ${{ steps.versioning.outputs.version }}
+          prerelease: ${{ github.ref != 'refs/heads/main' }}
+          bodyFile: 'cli/bin/README.md'

Dockerfile

@@ -1,4 +1,8 @@
-FROM node:16-slim AS site-build
+ARG BUILDARCH=amd64
+ARG TARGETARCH=amd64
+
+### Front-end
+FROM node:16 AS front-end
 
 WORKDIR /app/ui-build
 
@@ -9,53 +13,81 @@ COPY ui .
 RUN npm run build
 RUN npm run build-ent
 
-FROM golang:1.16-alpine AS builder
-# Set necessary environment variables needed for our image.
-ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
+### Base builder image for native builds architecture
+FROM golang:1.16-alpine AS builder-native-base
+ENV CGO_ENABLED=1 GOOS=linux
+RUN apk add libpcap-dev g++
 
-RUN apk add libpcap-dev gcc g++ make bash perl-utils
 
-# Move to agent working directory (/agent-build).
+### Intermediate builder image for x86-64 to x86-64 native builds
+FROM builder-native-base AS builder-from-amd64-to-amd64
+ENV GOARCH=amd64
+
+
+### Intermediate builder image for AArch64 to AArch64 native builds
+FROM builder-native-base AS builder-from-arm64v8-to-arm64v8
+ENV GOARCH=arm64
+
+
+### Builder image for x86-64 to AArch64 cross-compilation
+FROM up9inc/linux-arm64-musl-go-libpcap AS builder-from-amd64-to-arm64v8
+ENV CGO_ENABLED=1 GOOS=linux
+ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap"
+
+
+### Final builder image where the build happens
+# Possible build strategies:
+# BUILDARCH=amd64 TARGETARCH=amd64
+# BUILDARCH=arm64v8 TARGETARCH=arm64v8
+# BUILDARCH=amd64 TARGETARCH=arm64v8
+ARG BUILDARCH=amd64
+ARG TARGETARCH=amd64
+FROM builder-from-${BUILDARCH}-to-${TARGETARCH} AS builder
+
+# Move to agent working directory (/agent-build)
 WORKDIR /app/agent-build
 
 COPY agent/go.mod agent/go.sum ./
 COPY shared/go.mod shared/go.mod ../shared/
 COPY tap/go.mod tap/go.mod ../tap/
-COPY tap/api/go.* ../tap/api/
+COPY tap/api/go.mod ../tap/api/
+COPY tap/extensions/amqp/go.mod ../tap/extensions/amqp/
+COPY tap/extensions/http/go.mod ../tap/extensions/http/
+COPY tap/extensions/kafka/go.mod ../tap/extensions/kafka/
+COPY tap/extensions/redis/go.mod ../tap/extensions/redis/
 RUN go mod download
-# cheap trick to make the build faster (As long as go.mod wasn't changes)
+# cheap trick to make the build faster (as long as go.mod did not change)
 RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
 
+# Copy and build agent code
+COPY shared ../shared
+COPY tap ../tap
+COPY agent .
+
 ARG COMMIT_HASH
 ARG GIT_BRANCH
 ARG BUILD_TIMESTAMP
 ARG SEM_VER=0.0.0
 
-# Copy and build agent code
-COPY shared ../shared
-COPY tap ../tap
-COPY agent .
-RUN go build -ldflags="-s -w \
-     -X 'mizuserver/pkg/version.GitCommitHash=${COMMIT_HASH}' \
-     -X 'mizuserver/pkg/version.Branch=${GIT_BRANCH}' \
-     -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
-     -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
+WORKDIR /app/agent-build
 
-COPY devops/build_extensions.sh ..
-RUN cd .. && /bin/bash build_extensions.sh
+RUN go build -ldflags="-extldflags=-static -s -w \
+    -X 'mizuserver/pkg/version.GitCommitHash=${COMMIT_HASH}' \
+    -X 'mizuserver/pkg/version.Branch=${GIT_BRANCH}' \
+    -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
+    -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
 
-FROM alpine:3.15
 
-RUN apk add bash libpcap-dev
+### The shipped image
+ARG TARGETARCH=amd64
+FROM ${TARGETARCH}/busybox:latest
 
 WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
-COPY --from=builder ["/app/agent/build/extensions", "extensions"]
-COPY --from=site-build ["/app/ui-build/build", "site"]
-COPY --from=site-build ["/app/ui-build/build-ent", "site-standalone"]
-RUN mkdir /app/data/
+COPY --from=front-end ["/app/ui-build/build", "site"]
+COPY --from=front-end ["/app/ui-build/build-ent", "site-standalone"]
 
 # gin-gonic runs in debug mode without this
 ENV GIN_MODE=release

Makefile

@@ -8,7 +8,7 @@ SHELL=/bin/bash
 # HELP
 # This will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help ui extensions extensions-debug agent agent-debug cli tap docker
+.PHONY: help ui agent agent-debug cli tap docker
 
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -37,13 +37,11 @@ build-cli-ci: ## Build CLI for CI.
 agent: ## Build agent.
 	@(echo "building mizu agent .." )
 	@(cd agent; go build -o build/mizuagent main.go)
-	${MAKE} extensions
 	@ls -l agent/build
 
 agent-debug: ## Build agent for debug.
 	@(echo "building mizu agent for debug.." )
 	@(cd agent; go build -gcflags="all=-N -l" -o build/mizuagent main.go)
-	${MAKE} extensions-debug
 	@ls -l agent/build
 
 docker: ## Build and publish agent docker image.
@@ -55,10 +53,6 @@ push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
 	devops/build-push-featurebranch.sh
 
-push-docker-debug:
-	@echo "publishing debug Docker image .. "
-	devops/build-push-featurebranch-debug.sh
-
 build-docker-ci: ## Build agent docker image for CI.
 	@echo "building docker image for ci"
 	devops/build-agent-ci.sh
@@ -71,7 +65,7 @@ push-cli: ## Build and publish CLI.
 	gsutil cp -r ./cli/bin/* gs://${BUCKET_PATH}/
 	gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
 
-clean: clean-ui clean-agent clean-cli clean-docker clean-extensions ## Clean all build artifacts.
+clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
 
 clean-ui: ## Clean UI.
 	@(rm -rf ui/build ; echo "UI cleanup done" )
@@ -82,18 +76,9 @@ clean-agent: ## Clean agent.
 clean-cli:  ## Clean CLI.
 	@(cd cli; make clean ; echo "CLI cleanup done" )
 
-clean-extensions:  ## Clean extensions
-	@(rm -rf tap/extensions/*.so ; echo "Extensions cleanup done" )
-
 clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
-extensions-debug:
-	devops/build_extensions_debug.sh
-
-extensions:
-	devops/build_extensions.sh
-
 test-cli:
 	@echo "running cli tests"; cd cli && $(MAKE) test
 

agent/README.md

@@ -1,20 +0,0 @@
-# mizu agent
-Agent for MIZU (API server and tapper)
-Basic APIs:
-* /stats - retrieve statistics of collected data
-* /viewer - web ui
-
-## Remote Debugging
-### Setup remote debugging
-1. Run `go get github.com/go-delve/delve/cmd/dlv`
-2. Create a "Go Remote" run/debug configuration in Intellij, set to localhost:2345
-3. Build and push a debug image using
-   `docker build . -t  gcr.io/up9-docker-hub/mizu/debug:latest -f debug.Dockerfile && docker push gcr.io/up9-docker-hub/mizu/debug:latest`
-
-### Connecting
-1. Start mizu using the cli with the debug
-   image `mizu tap --set agent-image=gcr.io/up9-docker-hub/mizu/debug:latest {tapped_pod_name}`
-2. Forward the debug port using `kubectl port-forward -n default mizu-api-server 2345:2345`
-3. Run the run/debug configuration you've created earlier in Intellij.
-
-<small>Do note that dlv won't start the api until a debugger connects to it.</small>

agent/go.mod

@@ -21,10 +21,14 @@ require (
 	github.com/ory/kratos-client-go v0.8.2-alpha.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/stretchr/testify v1.7.0
-	github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d
+	github.com/up9inc/basenine/client/go v0.0.0-20220125035724-573fff0d5075
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
+	github.com/up9inc/mizu/tap/extensions/amqp v0.0.0
+	github.com/up9inc/mizu/tap/extensions/http v0.0.0
+	github.com/up9inc/mizu/tap/extensions/kafka v0.0.0
+	github.com/up9inc/mizu/tap/extensions/redis v0.0.0
 	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
 	k8s.io/api v0.21.2
 	k8s.io/apimachinery v0.21.2
@@ -36,3 +40,11 @@ replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 replace github.com/up9inc/mizu/tap v0.0.0 => ../tap
 
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api
+
+replace github.com/up9inc/mizu/tap/extensions/amqp v0.0.0 => ../tap/extensions/amqp
+
+replace github.com/up9inc/mizu/tap/extensions/http v0.0.0 => ../tap/extensions/http
+
+replace github.com/up9inc/mizu/tap/extensions/kafka v0.0.0 => ../tap/extensions/kafka
+
+replace github.com/up9inc/mizu/tap/extensions/redis v0.0.0 => ../tap/extensions/redis

agent/go.sum

@@ -77,6 +77,8 @@ github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:o
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg=
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
+github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
+github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
@@ -122,6 +124,8 @@ github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4Kfc
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -136,10 +140,13 @@ github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi
 github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
 github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
+github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=
+github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
@@ -337,6 +344,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -436,11 +444,16 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.14.1 h1:hLQYb23E8/fO+1u53d02A97a8UnsddcvYzq4ERRU4ds=
+github.com/klauspost/compress v1.14.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -502,6 +515,8 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/ohler55/ojg v1.12.12 h1:hepbQFn7GHAecTPmwS3j5dCiOLsOpzPLvhiqnlAVAoE=
+github.com/ohler55/ojg v1.12.12/go.mod h1:LBbIVRAgoFbYBXQhRhuEpaJIqq+goSO63/FQ+nyJU88=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
@@ -529,6 +544,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A=
+github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -564,6 +581,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 h1:TToq11gyfNlrMFZiYujSekIsPd9AmsA2Bj/iv+s4JHE=
 github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/segmentio/kafka-go v0.4.27 h1:sIhEozeL/TLN2mZ5dkG462vcGEWYKS+u31sXPjKhAM4=
+github.com/segmentio/kafka-go v0.4.27/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -615,8 +634,8 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d h1:WTz53dcfqCIWZpZLQoHbIcNc21s0ZHEZH7EqMPp99qQ=
-github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20220125035724-573fff0d5075 h1:Tp0yckZkvb8BNC+bB4B+fBuQdomTner6bOte1S8SYCo=
+github.com/up9inc/basenine/client/go v0.0.0-20220125035724-573fff0d5075/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
@@ -627,6 +646,7 @@ github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
@@ -662,6 +682,7 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

agent/main.go

@@ -19,9 +19,6 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
-	"path"
-	"path/filepath"
-	"plugin"
 	"sort"
 	"strconv"
 	"strings"
@@ -40,6 +37,11 @@ import (
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap"
 	tapApi "github.com/up9inc/mizu/tap/api"
+
+	amqpExt "github.com/up9inc/mizu/tap/extensions/amqp"
+	httpExt "github.com/up9inc/mizu/tap/extensions/http"
+	kafkaExt "github.com/up9inc/mizu/tap/extensions/kafka"
+	redisExt "github.com/up9inc/mizu/tap/extensions/redis"
 )
 
 var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API")
@@ -189,36 +191,36 @@ func configureBasenineServer(host string, port string) {
 }
 
 func loadExtensions() {
-	dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
-	extensionsDir := path.Join(dir, "./extensions/")
-
-	files, err := ioutil.ReadDir(extensionsDir)
-	if err != nil {
-		logger.Log.Fatal(err)
-	}
-	extensions = make([]*tapApi.Extension, len(files))
+	extensions = make([]*tapApi.Extension, 4)
 	extensionsMap = make(map[string]*tapApi.Extension)
-	for i, file := range files {
-		filename := file.Name()
-		logger.Log.Infof("Loading extension: %s", filename)
-		extension := &tapApi.Extension{
-			Path: path.Join(extensionsDir, filename),
-		}
-		plug, _ := plugin.Open(extension.Path)
-		extension.Plug = plug
-		symDissector, err := plug.Lookup("Dissector")
 
-		var dissector tapApi.Dissector
-		var ok bool
-		dissector, ok = symDissector.(tapApi.Dissector)
-		if err != nil || !ok {
-			panic(fmt.Sprintf("Failed to load the extension: %s", extension.Path))
-		}
-		dissector.Register(extension)
-		extension.Dissector = dissector
-		extensions[i] = extension
-		extensionsMap[extension.Protocol.Name] = extension
-	}
+	extensionAmqp := &tapApi.Extension{}
+	dissectorAmqp := amqpExt.NewDissector()
+	dissectorAmqp.Register(extensionAmqp)
+	extensionAmqp.Dissector = dissectorAmqp
+	extensions[0] = extensionAmqp
+	extensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
+
+	extensionHttp := &tapApi.Extension{}
+	dissectorHttp := httpExt.NewDissector()
+	dissectorHttp.Register(extensionHttp)
+	extensionHttp.Dissector = dissectorHttp
+	extensions[1] = extensionHttp
+	extensionsMap[extensionHttp.Protocol.Name] = extensionHttp
+
+	extensionKafka := &tapApi.Extension{}
+	dissectorKafka := kafkaExt.NewDissector()
+	dissectorKafka.Register(extensionKafka)
+	extensionKafka.Dissector = dissectorKafka
+	extensions[2] = extensionKafka
+	extensionsMap[extensionKafka.Protocol.Name] = extensionKafka
+
+	extensionRedis := &tapApi.Extension{}
+	dissectorRedis := redisExt.NewDissector()
+	dissectorRedis.Register(extensionRedis)
+	extensionRedis.Dissector = dissectorRedis
+	extensions[3] = extensionRedis
+	extensionsMap[extensionRedis.Protocol.Name] = extensionRedis
 
 	sort.Slice(extensions, func(i, j int) bool {
 		return extensions[i].Protocol.Priority < extensions[j].Protocol.Priority

cli/Makefile

@@ -30,15 +30,12 @@ build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables
 build-all: ## Build for all supported platforms.
 	@echo "Compiling for every OS and Platform"
 	@mkdir -p bin && sed s/_SEM_VER_/$(SEM_VER)/g README.md.TEMPLATE >  bin/README.md
-	@$(MAKE) build GOOS=darwin GOARCH=amd64
 	@$(MAKE) build GOOS=linux GOARCH=amd64
+	@$(MAKE) build GOOS=linux GOARCH=arm64
+	@$(MAKE) build GOOS=darwin GOARCH=amd64
 	@$(MAKE) build GOOS=darwin GOARCH=arm64
 	@$(MAKE) build GOOS=windows GOARCH=amd64
 	@mv ./bin/mizu_windows_amd64 ./bin/mizu.exe
-	@# $(MAKE) GOOS=linux GOARCH=386
-	@# $(MAKE) GOOS=windows GOARCH=386
-	@# $(MAKE) GOOS=linux GOARCH=arm64
-	@# $(MAKE) GOOS=windows GOARCH=arm64
 	@echo "---------"
 	@find ./bin -ls
 

cli/README.md.TEMPLATE

@@ -3,22 +3,27 @@ Full changelog for stable release see in [docs](https://github.com/up9inc/mizu/b
 
 ## Download Mizu for your platform
 
-**Mac** (Intel)
+**Mac** (x86-64/Intel)
 ```
 curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_darwin_amd64 && chmod 755 mizu
 ```
 
-**Mac** (Apple M1 silicon)
+**Mac** (AArch64/Apple M1 silicon)
 ```
 curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_darwin_arm64 && chmod 755 mizu
 ```
 
-**Linux** 
+**Linux** (x86-64)
 ```
 curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_linux_amd64 && chmod 755 mizu
 ```
 
-**Windows** (Intel 64bit)
+**Linux** (AArch64)
+```
+curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_linux_arm64 && chmod 755 mizu
+```
+
+**Windows** (x86-64)
 ```
 curl -LO https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu.exe
 ```

cli/config/configStruct.go

@@ -55,7 +55,7 @@ func (config *ConfigStruct) SetDefaults() {
 	config.BasenineImage = fmt.Sprintf("%s:%s", shared.BasenineImageRepo, shared.BasenineImageTag)
 	config.KratosImage = shared.KratosImageDefault
 	config.KetoImage = shared.KetoImageDefault
-	config.AgentImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", mizu.Branch, mizu.SemVer)
+	config.AgentImage = fmt.Sprintf("%s:%s", shared.MizuAgentImageRepo, mizu.SemVer)
 	config.ConfigFilePath = path.Join(mizu.GetMizuFolderPath(), "config.yaml")
 }
 

debug.Dockerfile

@@ -1,69 +0,0 @@
-# creates image in which mizu agent is remotely debuggable using delve
-FROM node:16-slim AS site-build
-
-WORKDIR /app/ui-build
-
-COPY ui/package.json .
-COPY ui/package-lock.json .
-RUN npm i
-COPY ui .
-RUN npm run build
-RUN npm run build-ent
-
-FROM golang:1.16-alpine AS builder
-# Set necessary environment variables needed for our image.
-ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
-
-RUN apk add libpcap-dev gcc g++ make bash perl-utils
-
-# Move to agent working directory (/agent-build).
-WORKDIR /app/agent-build
-
-COPY agent/go.mod agent/go.sum ./
-COPY shared/go.mod shared/go.mod ../shared/
-COPY tap/go.mod tap/go.mod ../tap/
-COPY tap/api/go.* ../tap/api/
-RUN go mod download
-# cheap trick to make the build faster (As long as go.mod wasn't changes)
-RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
-
-ARG COMMIT_HASH
-ARG GIT_BRANCH
-ARG BUILD_TIMESTAMP
-ARG SEM_VER=0.0.0
-
-# Copy and build agent code
-COPY shared ../shared
-COPY tap ../tap
-COPY agent .
-# Include gcflags for debugging
-RUN go build -gcflags="all=-N -l" -o mizuagent .
-
-COPY devops/build_extensions_debug.sh ..
-RUN cd .. && /bin/bash build_extensions_debug.sh
-
-FROM golang:1.16-alpine
-
-# Set necessary environment variables needed for our image.
-RUN apk add bash libpcap-dev gcc g++
-
-WORKDIR /app
-
-# Copy binary and config files from /build to root folder of scratch container.
-COPY --from=builder ["/app/agent-build/mizuagent", "."]
-COPY --from=builder ["/app/agent/build/extensions", "extensions"]
-COPY --from=site-build ["/app/ui-build/build", "site"]
-COPY --from=site-build ["/app/ui-build/build-ent", "site-standalone"]
-RUN mkdir /app/data/
-
-# install delve
-ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
-RUN go get github.com/go-delve/delve/cmd/dlv
-
-ENV GIN_MODE=debug
-
-# delve ports
-EXPOSE 2345 2346
-
-# this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
-ENTRYPOINT "/app/mizuagent"

devops/build-push-featurebranch-debug.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-set -e
-
-GCP_PROJECT=up9-docker-hub
-REPOSITORY=gcr.io/$GCP_PROJECT
-SERVER_NAME=mizu
-GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
-
-DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
-SEM_VER=${SEM_VER=0.0.0}
-
-DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
-
-if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
-then
-  echo "Pushing to $GIT_BRANCH is allowed only via CI"
-  exit 1
-fi
-
-echo "building ${DOCKER_TAGGED_BUILDS[@]}"
-DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
-docker build -f debug.Dockerfile $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
-
-for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
-do
-  echo pushing "$DOCKER_TAG"
-  docker push "$DOCKER_TAG"
-done

devops/build_extensions.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-for f in tap/extensions/*; do
-    if [ -d "$f" ]; then
-        echo Building extension: $f
-        extension=$(basename $f) && \
-        cd tap/extensions/${extension} && \
-        go build -buildmode=plugin -o ../${extension}.so .  && \
-        cd ../../..  && \
-        mkdir -p agent/build/extensions  && \
-        cp tap/extensions/${extension}.so agent/build/extensions
-    fi
-done

devops/build_extensions_debug.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-for f in tap/extensions/*; do
-    if [ -d "$f" ]; then
-        extension=$(basename $f) && \
-        cd tap/extensions/${extension} && \
-        go build -gcflags="all=-N -l" -buildmode=plugin -o ../${extension}.so .  && \
-        cd ../../..  && \
-        mkdir -p agent/build/extensions  && \
-        cp tap/extensions/${extension}.so agent/build/extensions
-    fi
-done

devops/linux-arm64-musl-go-libpcap/Dockerfile

@@ -0,0 +1,18 @@
+FROM dockcross/linux-arm64-musl:latest AS builder-from-amd64-to-arm64v8
+
+# Install Go
+RUN curl https://go.dev/dl/go1.16.13.linux-amd64.tar.gz -Lo ./go.linux-amd64.tar.gz
+RUN curl https://go.dev/dl/go1.16.13.linux-amd64.tar.gz.asc -Lo ./go.linux-amd64.tar.gz.asc
+RUN curl https://dl.google.com/dl/linux/linux_signing_key.pub  -Lo linux_signing_key.pub
+RUN gpg --import linux_signing_key.pub && gpg --verify ./go.linux-amd64.tar.gz.asc ./go.linux-amd64.tar.gz
+RUN rm -rf /usr/local/go && tar -C /usr/local -xzf go.linux-amd64.tar.gz
+ENV PATH "$PATH:/usr/local/go/bin"
+
+# Compile libpcap from source
+RUN curl https://www.tcpdump.org/release/libpcap-1.10.1.tar.gz -Lo ./libpcap.tar.gz
+RUN curl https://www.tcpdump.org/release/libpcap-1.10.1.tar.gz.sig -Lo ./libpcap.tar.gz.sig
+RUN curl https://www.tcpdump.org/release/signing-key.asc -Lo ./signing-key.asc
+RUN gpg --import signing-key.asc && gpg --verify libpcap.tar.gz.sig libpcap.tar.gz
+RUN tar -xzf libpcap.tar.gz && mv ./libpcap-* ./libpcap
+RUN cd ./libpcap && ./configure --host=arm && make
+RUN cp /work/libpcap/libpcap.a /usr/xcc/aarch64-linux-musl-cross/lib/gcc/aarch64-linux-musl/*/

devops/linux-arm64-musl-go-libpcap/build-push.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+docker build . -t up9inc/linux-arm64-musl-go-libpcap && docker push up9inc/linux-arm64-musl-go-libpcap

shared/consts.go

@@ -14,10 +14,11 @@ const (
 	GoGCEnvVar                       = "GOGC"
 	DefaultApiServerPort             = 8899
 	LogLevelEnvVar                   = "LOG_LEVEL"
+	MizuAgentImageRepo               = "docker.io/up9inc/mizu"
 	BasenineHost                     = "127.0.0.1"
 	BaseninePort                     = "9099"
-	BasenineImageRepo                = "ghcr.io/up9inc/basenine"
-	BasenineImageTag                 = "v0.3.0"
+	BasenineImageRepo                = "docker.io/up9inc/basenine"
+	BasenineImageTag                 = "v0.4.13"
 	KratosImageDefault               = "gcr.io/up9-docker-hub/mizu-kratos/stable:0.0.0"
 	KetoImageDefault                 = "gcr.io/up9-docker-hub/mizu-keto/stable:0.0.0"
 )

tap/api/api.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"plugin"
 	"sync"
 	"time"
 
@@ -38,7 +37,6 @@ type TCP struct {
 type Extension struct {
 	Protocol   *Protocol
 	Path       string
-	Plug       *plugin.Plugin
 	Dissector  Dissector
 	MatcherMap *sync.Map
 }

tap/extensions/amqp/helpers.go

@@ -1,4 +1,4 @@
-package main
+package amqp
 
 import (
 	"encoding/json"

tap/extensions/amqp/main.go

@@ -1,4 +1,4 @@
-package main
+package amqp
 
 import (
 	"bufio"
@@ -328,3 +328,7 @@ func (d dissecting) Macros() map[string]string {
 }
 
 var Dissector dissecting
+
+func NewDissector() api.Dissector {
+	return Dissector
+}

tap/extensions/amqp/read.go

@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 // Source code and contact info at http://github.com/streadway/amqp
 
-package main
+package amqp
 
 import (
 	"bytes"

tap/extensions/amqp/spec091.go

@@ -6,7 +6,7 @@
 /* GENERATED FILE - DO NOT EDIT */
 /* Rebuild from the spec/gen.go tool */
 
-package main
+package amqp
 
 import (
 	"encoding/binary"

tap/extensions/amqp/structs.go

@@ -1,4 +1,4 @@
-package main
+package amqp
 
 import (
 	"encoding/json"

tap/extensions/amqp/types.go

@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 // Source code and contact info at http://github.com/streadway/amqp
 
-package main
+package amqp
 
 import (
 	"fmt"

tap/extensions/amqp/write.go

@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 // Source code and contact info at http://github.com/streadway/amqp
 
-package main
+package amqp
 
 import (
 	"bufio"

tap/extensions/http/handlers.go

@@ -1,4 +1,4 @@
-package main
+package http
 
 import (
 	"bufio"

tap/extensions/http/helpers.go

@@ -1,4 +1,4 @@
-package main
+package http
 
 import (
 	"encoding/json"

tap/extensions/http/http2_assembler.go

@@ -1,4 +1,4 @@
-package main
+package http
 
 import (
 	"bufio"

tap/extensions/http/main.go

@@ -1,4 +1,4 @@
-package main
+package http
 
 import (
 	"bufio"
@@ -464,3 +464,7 @@ func (d dissecting) Macros() map[string]string {
 }
 
 var Dissector dissecting
+
+func NewDissector() api.Dissector {
+	return Dissector
+}

tap/extensions/http/matcher.go

@@ -1,4 +1,4 @@
-package main
+package http
 
 import (
 	"fmt"

tap/extensions/http/sensitive_data_cleaner.go

@@ -1,4 +1,4 @@
-package main
+package http
 
 import (
 	"bytes"

tap/extensions/kafka/buffer.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"bytes"

tap/extensions/kafka/cluster.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"fmt"

tap/extensions/kafka/compression.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"errors"

tap/extensions/kafka/decode.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"bytes"

tap/extensions/kafka/discard.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import "bufio"
 

tap/extensions/kafka/encode.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"bytes"

tap/extensions/kafka/error.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"fmt"

tap/extensions/kafka/go.mod

@@ -4,8 +4,9 @@ go 1.16
 
 require (
 	github.com/fatih/camelcase v1.0.0
+	github.com/klauspost/compress v1.14.1 // indirect
 	github.com/ohler55/ojg v1.12.12
-	github.com/segmentio/kafka-go v0.4.17
+	github.com/segmentio/kafka-go v0.4.27
 	github.com/up9inc/mizu/tap/api v0.0.0
 )
 

tap/extensions/kafka/go.sum

@@ -11,8 +11,9 @@ github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
 github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.14.1 h1:hLQYb23E8/fO+1u53d02A97a8UnsddcvYzq4ERRU4ds=
+github.com/klauspost/compress v1.14.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -23,8 +24,8 @@ github.com/ohler55/ojg v1.12.12/go.mod h1:LBbIVRAgoFbYBXQhRhuEpaJIqq+goSO63/FQ+n
 github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A=
 github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/segmentio/kafka-go v0.4.17 h1:IyqRstL9KUTDb3kyGPOOa5VffokKWSEzN6geJ92dSDY=
-github.com/segmentio/kafka-go v0.4.17/go.mod h1:19+Eg7KwrNKy/PFhiIthEPkO8k+ac7/ZYXwYM9Df10w=
+github.com/segmentio/kafka-go v0.4.27 h1:sIhEozeL/TLN2mZ5dkG462vcGEWYKS+u31sXPjKhAM4=
+github.com/segmentio/kafka-go v0.4.27/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=

tap/extensions/kafka/helpers.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"encoding/json"

tap/extensions/kafka/main.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"bufio"
@@ -223,3 +223,7 @@ func (d dissecting) Macros() map[string]string {
 }
 
 var Dissector dissecting
+
+func NewDissector() api.Dissector {
+	return Dissector
+}

tap/extensions/kafka/matcher.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"sync"

tap/extensions/kafka/protocol.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"fmt"

tap/extensions/kafka/protocol_make.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"encoding/binary"

tap/extensions/kafka/read.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"bufio"

tap/extensions/kafka/record.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"encoding/binary"

tap/extensions/kafka/record_bytes.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"github.com/segmentio/kafka-go/protocol"

tap/extensions/kafka/reflect.go

@@ -1,6 +1,6 @@
 // +build !unsafe
 
-package main
+package kafka
 
 import (
 	"reflect"

tap/extensions/kafka/request.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"fmt"

tap/extensions/kafka/response.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"fmt"

tap/extensions/kafka/structs.go

@@ -1,4 +1,4 @@
-package main
+package kafka
 
 import (
 	"time"

tap/extensions/redis/errors.go

@@ -1,4 +1,4 @@
-package main
+package redis
 
 //ConnectError redis connection error,such as io timeout
 type ConnectError struct {

tap/extensions/redis/handlers.go

@@ -1,4 +1,4 @@
-package main
+package redis
 
 import (
 	"fmt"

tap/extensions/redis/helpers.go

@@ -1,4 +1,4 @@
-package main
+package redis
 
 import (
 	"encoding/json"

tap/extensions/redis/main.go

@@ -1,4 +1,4 @@
-package main
+package redis
 
 import (
 	"bufio"
@@ -124,3 +124,7 @@ func (d dissecting) Macros() map[string]string {
 }
 
 var Dissector dissecting
+
+func NewDissector() api.Dissector {
+	return Dissector
+}

tap/extensions/redis/matcher.go

@@ -1,4 +1,4 @@
-package main
+package redis
 
 import (
 	"fmt"

tap/extensions/redis/read.go

@@ -1,4 +1,4 @@
-package main
+package redis
 
 import (
 	"bufio"

tap/extensions/redis/structs.go

@@ -1,4 +1,4 @@
-package main
+package redis
 
 type RedisType string
 type RedisCommand string