Bring back Set up Cloud SDK step to Build the CLI and publish job

Also revert changes in the shell scripts

.dockerignore

@@ -2,6 +2,7 @@
 .dockerignore
 .editorconfig
 .gitignore
+**/.env*
 Dockerfile
 Makefile
 LICENSE

.github/workflows/acceptance_tests.yml

@@ -8,31 +8,25 @@ on:
     branches:
       - 'develop'
 
+concurrency:
+  group: mizu-acceptance-tests-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   run-acceptance-tests:
     name: Run acceptance tests
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.17
+      - name: Set up Go 1.16
         uses: actions/setup-go@v2
         with:
-          go-version: '^1.17'
+          go-version: '^1.16'
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
 
       - name: Setup acceptance test
-        run: ./acceptanceTests/setup.sh
-
-      - name: Create k8s users and change context
-        env:
-          USERNAME_UNRESTRICTED: user-with-clusterwide-access
-          USERNAME_RESTRICTED: user-with-restricted-access
-        run: |
-          ./acceptanceTests/create_user.sh "${USERNAME_UNRESTRICTED}"
-          ./acceptanceTests/create_user.sh "${USERNAME_RESTRICTED}"
-          kubectl apply -f cli/cmd/permissionFiles/permissions-all-namespaces-tap.yaml
-          kubectl config use-context ${USERNAME_UNRESTRICTED}
+        run: source ./acceptanceTests/setup.sh
 
       - name: Test
         run: make acceptance-test
@@ -43,7 +37,7 @@ jobs:
         with:
           status: ${{ job.status }}
           notification_title: 'Mizu {workflow} has {status_message}'
-          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit <{commit_url}|{commit_sha}> by ${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}> ```${{ github.event.head_commit.message }}```'
+          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit: <{commit_url}|{commit_sha}>'
           footer: 'Linked Repo <{repo_url}|{repo}>'
           notify_when: 'failure'
         env:

.github/workflows/build-custom-branch.yml

@@ -1,44 +0,0 @@
-name: Build Custom Branch
-
-on: push
-
-concurrency:
-  group: custom-branch-build-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    name: Push custom branch image to GCR
-    runs-on: ubuntu-latest
-    if: ${{ contains(github.event.head_commit.message, '#build_and_publish_custom_image') }}
-
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v2
-
-      - id: 'auth'
-        uses: 'google-github-actions/auth@v0'
-        with:
-          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
-
-      - name: 'Set up Cloud SDK'
-        uses: 'google-github-actions/setup-gcloud@v0'
-
-      - name: Get base image name
-        shell: bash
-        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
-        id: base_image_step
-
-      - name: Login to GCR
-        uses: docker/login-action@v1
-        with:
-          registry: gcr.io
-          username: _json_key
-          password: ${{ secrets.GCR_JSON_KEY }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: ${{ steps.base_image_step.outputs.image }}:latest

.github/workflows/build.yml

@@ -1,62 +0,0 @@
-name: Build
-
-on:
-  pull_request:
-    branches:
-      - 'develop'
-      - 'main'
-
-concurrency:
-  group: mizu-pr-validation-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-cli:
-    name: CLI executable build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 2
-      
-      - name: Check modified files
-        id: modified_files
-        run: devops/check_modified_files.sh cli/
-
-      - name: Set up Go 1.17
-        if: steps.modified_files.outputs.matched == 'true'
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.17'
-
-      - name: Build CLI
-        if: steps.modified_files.outputs.matched == 'true'
-        run: make cli
-
-  build-agent:
-    name: Agent docker image build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 2
-      
-      - name: Check modified files
-        id: modified_files
-        run: devops/check_modified_files.sh agent/ shared/ tap/ ui/ ui-common/ Dockerfile
-
-      - name: Set up Docker Buildx
-        if: steps.modified_files.outputs.matched == 'true'
-        uses: docker/setup-buildx-action@v1
-
-      - name: Build
-        uses: docker/build-push-action@v2
-        if: steps.modified_files.outputs.matched == 'true'
-        with:
-          context: .
-          push: false
-          tags: up9inc/mizu:devlatest
-          cache-from: type=gha
-          cache-to: type=gha,mode=max

.github/workflows/pr_validation.yml

@@ -0,0 +1,61 @@
+name: PR validation
+
+on:
+  pull_request:
+    branches:
+      - 'develop'
+      - 'main'
+
+concurrency:
+  group: mizu-pr-validation-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-cli:
+    name: Build CLI
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Build CLI
+        run: make cli
+
+  build-agent:
+    name: Build Agent
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - shell: bash
+        run: |
+          sudo apt-get install libpcap-dev
+
+      - name: Build Agent
+        run: make agent
+
+  build-ui:
+    name: Build UI
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Node 16
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Build UI
+        run: make ui

.github/workflows/publish.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - 'develop'
       - 'main'
+      - 'feature/multiarch_build'
 
 concurrency:
   group: mizu-publish-${{ github.ref }}
@@ -32,15 +33,14 @@ jobs:
         id: condval
         with:
           cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "stable"
-          if_false: "dev"
+          if_true: "minor"
+          if_false: "patch"
 
-      - name: Auto Increment Ver Action
-        uses: docker://igorgov/auto-inc-ver:v2.0.0
+      - name: Auto increment SemVer action
+        uses: MCKanpolat/auto-semver-action@1.0.5
         id: versioning
         with:
-          mode: ${{ steps.condval.outputs.value }}
-          suffix: 'dev'
+          releaseType: ${{ steps.condval.outputs.value }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Get version parameters
@@ -58,8 +58,6 @@ jobs:
             up9inc/mizu
           tags: |
             type=raw,${{ steps.versioning.outputs.version }}
-            type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
-            type=raw,value=dev-latest,enable=${{ steps.condval.outputs.value == 'dev' }}
           flavor: |
             latest=auto
             prefix=
@@ -79,8 +77,8 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
-            TARGETARCH=${{ matrix.target }}
-            VER=${{ steps.versioning.outputs.version }}
+            ARCH=${{ matrix.target }}
+            SEM_VER=${{ steps.versioning.outputs.version }}
             BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
             GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
             COMMIT_HASH=${{ github.sha }}
@@ -101,28 +99,25 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v2
 
-      - id: 'auth'
-        uses: 'google-github-actions/auth@v0'
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@master
         with:
-          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
-
-      - name: 'Set up Cloud SDK'
-        uses: 'google-github-actions/setup-gcloud@v0'
+          service_account_key: ${{ secrets.GCR_JSON_KEY }}
+          export_default_credentials: true
 
       - name: Determine versioning strategy
         uses: haya14busa/action-cond@v1
         id: condval
         with:
           cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "stable"
-          if_false: "dev"
+          if_true: "minor"
+          if_false: "patch"
 
-      - name: Auto Increment Ver Action
-        uses: docker://igorgov/auto-inc-ver:v2.0.0
+      - name: Auto increment SemVer action
+        uses: MCKanpolat/auto-semver-action@1.0.5
         id: versioning
         with:
-          mode: ${{ steps.condval.outputs.value }}
-          suffix: 'dev'
+          releaseType: ${{ steps.condval.outputs.value }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Get version parameters
@@ -145,8 +140,6 @@ jobs:
             ${{ steps.base_image_step.outputs.image }}
           tags: |
             type=raw,${{ steps.versioning.outputs.version }}
-            type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
-            type=raw,value=dev-latest,enable=${{ steps.condval.outputs.value == 'dev' }}
           flavor: |
             latest=auto
             prefix=
@@ -167,8 +160,8 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
-            TARGETARCH=${{ matrix.target }}
-            VER=${{ steps.versioning.outputs.version }}
+            ARCH=${{ matrix.target }}
+            SEM_VER=${{ steps.versioning.outputs.version }}
             BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
             GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
             COMMIT_HASH=${{ github.sha }}
@@ -183,15 +176,14 @@ jobs:
         id: condval
         with:
           cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "stable"
-          if_false: "dev"
+          if_true: "minor"
+          if_false: "patch"
 
-      - name: Auto Increment Ver Action
-        uses: docker://igorgov/auto-inc-ver:v2.0.0
+      - name: Auto increment SemVer action
+        uses: MCKanpolat/auto-semver-action@1.0.5
         id: versioning
         with:
-          mode: ${{ steps.condval.outputs.value }}
-          suffix: 'dev'
+          releaseType: ${{ steps.condval.outputs.value }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Get version parameters
@@ -209,8 +201,7 @@ jobs:
             up9inc/mizu
           tags: |
             type=raw,${{ steps.versioning.outputs.version }}
-            type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
-            type=raw,value=dev-latest,enable=${{ steps.condval.outputs.value == 'dev' }}
+
       - name: Login to Docker Hub
         uses: docker/login-action@v1
         with:
@@ -242,28 +233,24 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v2
 
-      - id: 'auth'
-        uses: 'google-github-actions/auth@v0'
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@master
         with:
-          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
+          service_account_key: ${{ secrets.GCR_JSON_KEY }}
+          export_default_credentials: true
 
-      - name: 'Set up Cloud SDK'
-        uses: 'google-github-actions/setup-gcloud@v0'
-
-      - name: Determine versioning strategy
-        uses: haya14busa/action-cond@v1
+      - uses: haya14busa/action-cond@v1
         id: condval
         with:
           cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "stable"
-          if_false: "dev"
+          if_true: "minor"
+          if_false: "patch"
 
-      - name: Auto Increment Ver Action
-        uses: docker://igorgov/auto-inc-ver:v2.0.0
+      - name: Auto Increment Semver Action
+        uses: MCKanpolat/auto-semver-action@1.0.5
         id: versioning
         with:
-          mode: ${{ steps.condval.outputs.value }}
-          suffix: 'dev'
+          releaseType: ${{ steps.condval.outputs.value }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Get version parameters
@@ -274,7 +261,7 @@ jobs:
         id: version_parameters
 
       - name: Build and Push CLI
-        run: make push-cli VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
+        run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
 
       - name: Log the version into a .txt file
         shell: bash

.github/workflows/static_code_analysis.yml

@@ -1,143 +0,0 @@
-name: Static code analysis
-
-on:
-  pull_request:
-    branches:
-      - 'develop'
-      - 'main'
-
-permissions:
-  contents: read
-
-jobs:
-  golangci:
-    name: Go lint
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 2
-
-      - uses: actions/setup-go@v2
-        with:
-            go-version: '^1.17'
-
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y libpcap-dev
-
-      - name: Check Agent modified files
-        id: agent_modified_files
-        run: devops/check_modified_files.sh agent/
-
-      - name: Go lint - agent
-        uses: golangci/golangci-lint-action@v2
-        if: steps.agent_modified_files.outputs.matched == 'true'
-        with:
-          version: latest
-          working-directory: agent
-          args: --timeout=3m
-
-      - name: Check shared modified files
-        id: shared_modified_files
-        run: devops/check_modified_files.sh shared/
-
-      - name: Go lint - shared
-        uses: golangci/golangci-lint-action@v2
-        if: steps.shared_modified_files.outputs.matched == 'true'
-        with:
-          version: latest
-          working-directory: shared
-          args: --timeout=3m
-
-      - name: Check tap modified files
-        id: tap_modified_files
-        run: devops/check_modified_files.sh tap/
-
-      - name: Go lint - tap
-        uses: golangci/golangci-lint-action@v2
-        if: steps.tap_modified_files.outputs.matched == 'true'
-        with:
-          version: latest
-          working-directory: tap
-          args: --timeout=3m
-
-      - name: Check cli modified files
-        id: cli_modified_files
-        run: devops/check_modified_files.sh cli/
-
-      - name: Go lint - CLI
-        uses: golangci/golangci-lint-action@v2
-        if: steps.cli_modified_files.outputs.matched == 'true'
-        with:
-          version: latest
-          working-directory: cli
-          args: --timeout=3m
-
-      - name: Check acceptanceTests modified files
-        id: acceptanceTests_modified_files
-        run: devops/check_modified_files.sh acceptanceTests/
-
-      - name: Go lint - acceptanceTests
-        uses: golangci/golangci-lint-action@v2
-        if: steps.acceptanceTests_modified_files.outputs.matched == 'true'
-        with:
-          version: latest
-          working-directory: acceptanceTests
-          args: --timeout=3m
-
-      - name: Check tap/api modified files
-        id: tap_api_modified_files
-        run: devops/check_modified_files.sh tap/api/
-
-      - name: Go lint - tap/api
-        uses: golangci/golangci-lint-action@v2
-        if: steps.tap_api_modified_files.outputs.matched == 'true'
-        with:
-          version: latest
-          working-directory: tap/api
-
-      - name: Check tap/extensions/amqp modified files
-        id: tap_amqp_modified_files
-        run: devops/check_modified_files.sh tap/extensions/amqp/
-
-      - name: Go lint - tap/extensions/amqp
-        uses: golangci/golangci-lint-action@v2
-        if: steps.tap_amqp_modified_files.outputs.matched == 'true'
-        with:
-          version: latest
-          working-directory: tap/extensions/amqp
-      
-      - name: Check tap/extensions/http modified files
-        id: tap_http_modified_files
-        run: devops/check_modified_files.sh tap/extensions/http/
-
-      - name: Go lint - tap/extensions/http
-        uses: golangci/golangci-lint-action@v2
-        if: steps.tap_http_modified_files.outputs.matched == 'true'
-        with:
-          version: latest
-          working-directory: tap/extensions/http
-
-      - name: Check tap/extensions/kafka modified files
-        id: tap_kafka_modified_files
-        run: devops/check_modified_files.sh tap/extensions/kafka/
-
-      - name: Go lint - tap/extensions/kafka
-        uses: golangci/golangci-lint-action@v2
-        if: steps.tap_kafka_modified_files.outputs.matched == 'true'
-        with:
-          version: latest
-          working-directory: tap/extensions/kafka
-
-      - name: Check tap/extensions/redis modified files
-        id: tap_redis_modified_files
-        run: devops/check_modified_files.sh tap/extensions/redis/
-
-      - name: Go lint - tap/extensions/redis
-        uses: golangci/golangci-lint-action@v2
-        if: steps.tap_redis_modified_files.outputs.matched == 'true'
-        with:
-          version: latest
-          working-directory: tap/extensions/redis

.github/workflows/test.yml

@@ -1,67 +0,0 @@
-name: Test
-
-on:
-  pull_request:
-    branches:
-      - 'develop'
-      - 'main'
-  push: # needed to upload test coverage report to codecov
-    branches:
-      - 'develop'
-      - 'main'
-
-concurrency:
-  group: mizu-tests-validation-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run-unit-tests:
-    name: Unit Tests
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 2
-
-      - name: Set up Go 1.17
-        uses: actions/setup-go@v2
-        with:
-          go-version: '^1.17'
-
-      - name: Install libpcap
-        shell: bash
-        run: |
-          sudo apt-get install libpcap-dev
-
-      - name: Check CLI modified files
-        id: cli_modified_files
-        run: devops/check_modified_files.sh cli/
-
-      - name: CLI Test
-        if: github.event_name == 'push' || steps.cli_modified_files.outputs.matched == 'true'
-        run: make test-cli
-
-      - name: Check Agent modified files
-        id: agent_modified_files
-        run: devops/check_modified_files.sh agent/
-
-      - name: Agent Test
-        if: github.event_name == 'push' || steps.agent_modified_files.outputs.matched == 'true'
-        run: make test-agent
-
-      - name: Shared Test
-        run: make test-shared
-      
-      - name: Check extensions modified files
-        id: ext_modified_files
-        run: devops/check_modified_files.sh tap/extensions/
-
-      - name: Extensions Test
-        if: github.event_name == 'push' || steps.ext_modified_files.outputs.matched == 'true'
-        run: make test-extensions
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v2
-

.github/workflows/tests_validation.yml

@@ -0,0 +1,56 @@
+name: tests validation
+
+on:
+  pull_request:
+    branches:
+      - 'develop'
+      - 'main'
+  push:
+    branches:
+      - 'develop'
+      - 'main'
+
+concurrency:
+  group: mizu-tests-validation-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-tests-cli:
+    name: Run CLI tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Test
+        run: make test-cli
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2
+
+  run-tests-agent:
+    name: Run Agent tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - shell: bash
+        run: |
+          sudo apt-get install libpcap-dev
+
+      - name: Test
+        run: make test-agent
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2

.gitignore

@@ -30,7 +30,7 @@ build
 pprof/*
 
 # Database Files
-*.db
+*.bin
 *.gob
 
 # Nohup Files - https://man7.org/linux/man-pages/man1/nohup.1p.html
@@ -44,12 +44,3 @@ cypress.env.json
 */cypress/screenshots
 */cypress/videos
 */cypress/support
-
-# Ignore test data in extensions
-tap/extensions/*/bin
-tap/extensions/*/expect
-
-# UI folders to ignore
-**/node_modules/**
-**/dist/**
-*.editorconfig

Dockerfile

@@ -1,69 +1,35 @@
-ARG BUILDARCH=amd64
-ARG TARGETARCH=amd64
-
-### Front-end common
-FROM node:16 AS front-end-common
-
-WORKDIR /app/ui-build
-COPY ui-common/package.json .
-COPY ui-common/package-lock.json .
-RUN npm i
-COPY ui-common .
-RUN npm pack
+ARG ARCH=amd64
 
 ### Front-end
 FROM node:16 AS front-end
 
 WORKDIR /app/ui-build
 
-COPY ui/package.json ui/package-lock.json ./
-COPY --from=front-end-common ["/app/ui-build/up9-mizu-common-0.0.0.tgz", "."]
+COPY ui/package.json .
+COPY ui/package-lock.json .
 RUN npm i
 COPY ui .
 RUN npm run build
 
-### Base builder image for native builds architecture
-FROM golang:1.17-alpine AS builder-native-base
-ENV CGO_ENABLED=1 GOOS=linux
-RUN apk add --no-cache libpcap-dev g++ perl-utils
 
+### Base of the builder image
+FROM golang:1.17-bullseye AS builder-base
 
-### Intermediate builder image for x86-64 to x86-64 native builds
-FROM builder-native-base AS builder-from-amd64-to-amd64
-ENV GOARCH=amd64
+# Set necessary environment variables needed for our image.
+ENV CGO_ENABLED=1 GOOS=linux GOARCH=${GOARCH}
 
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        libpcap-dev
 
-### Intermediate builder image for AArch64 to AArch64 native builds
-FROM builder-native-base AS builder-from-arm64v8-to-arm64v8
-ENV GOARCH=arm64
-
-
-### Builder image for x86-64 to AArch64 cross-compilation
-FROM up9inc/linux-arm64-musl-go-libpcap AS builder-from-amd64-to-arm64v8
-ENV CGO_ENABLED=1 GOOS=linux
-ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap"
-
-
-### Final builder image where the build happens
-# Possible build strategies:
-# BUILDARCH=amd64 TARGETARCH=amd64
-# BUILDARCH=arm64v8 TARGETARCH=arm64v8
-# BUILDARCH=amd64 TARGETARCH=arm64v8
-ARG BUILDARCH=amd64
-ARG TARGETARCH=amd64
-FROM builder-from-${BUILDARCH}-to-${TARGETARCH} AS builder
-
-# Move to agent working directory (/agent-build)
+# Move to agent working directory (/agent-build).
 WORKDIR /app/agent-build
 
 COPY agent/go.mod agent/go.sum ./
 COPY shared/go.mod shared/go.mod ../shared/
 COPY tap/go.mod tap/go.mod ../tap/
-COPY tap/api/go.mod ../tap/api/
-COPY tap/extensions/amqp/go.mod ../tap/extensions/amqp/
-COPY tap/extensions/http/go.mod ../tap/extensions/http/
-COPY tap/extensions/kafka/go.mod ../tap/extensions/kafka/
-COPY tap/extensions/redis/go.mod ../tap/extensions/redis/
+COPY tap/api/go.* ../tap/api/
 RUN go mod download
 # cheap trick to make the build faster (as long as go.mod did not change)
 RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
@@ -73,40 +39,61 @@ COPY shared ../shared
 COPY tap ../tap
 COPY agent .
 
+
+### Intermediate builder image for AMD64 architecture
+FROM builder-base AS builder-amd64
+
+ENV GOARCH=amd64
+
+
+### Intermediate builder image for ARM64 architecture
+FROM builder-base AS builder-arm64v8
+
+ENV GOARCH=arm64
+ENV CC=aarch64-linux-gnu-gcc
+ENV PKG_CONFIG_PATH=/usr/lib/aarch64-linux-gnu/pkgconfig
+
+RUN dpkg --add-architecture arm64 \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        gcc-aarch64-linux-gnu \
+        libpcap-dev:arm64
+
+
+### Final builder image where the building happens
+FROM builder-${ARCH} AS builder
+
 ARG COMMIT_HASH
 ARG GIT_BRANCH
 ARG BUILD_TIMESTAMP
-ARG VER=0.0
+ARG SEM_VER=0.0.0
 
 WORKDIR /app/agent-build
 
-RUN go build -ldflags="-extldflags=-static -s -w \
-    -X 'github.com/up9inc/mizu/agent/pkg/version.GitCommitHash=${COMMIT_HASH}' \
-    -X 'github.com/up9inc/mizu/agent/pkg/version.Branch=${GIT_BRANCH}' \
-    -X 'github.com/up9inc/mizu/agent/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
-    -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
+RUN go build -ldflags="-extldflags '-fuse-ld=bfd' -s -w \
+    -X 'mizuserver/pkg/version.GitCommitHash=${COMMIT_HASH}' \
+    -X 'mizuserver/pkg/version.Branch=${GIT_BRANCH}' \
+    -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
+    -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
+
+COPY devops/build_extensions.sh ..
+RUN cd .. && /bin/bash build_extensions.sh
 
-# Download Basenine executable, verify the sha1sum
-ADD https://github.com/up9inc/basenine/releases/download/v0.7.1/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
-ADD https://github.com/up9inc/basenine/releases/download/v0.7.1/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
 
-RUN shasum -a 256 -c basenine_linux_"${GOARCH}".sha256 && \
-    chmod +x ./basenine_linux_"${GOARCH}" && \
-    mv ./basenine_linux_"${GOARCH}" ./basenine
 
 ### The shipped image
-ARG TARGETARCH=amd64
-FROM ${TARGETARCH}/busybox:latest
-# gin-gonic runs in debug mode without this
-ENV GIN_MODE=release
+ARG ARCH=amd64
+FROM up9inc/debian-pcap:stable-slim-${ARCH}
 
-WORKDIR /app/data/
 WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
-COPY --from=builder ["/app/agent-build/basenine", "/usr/local/bin/basenine"]
+COPY --from=builder ["/app/agent/build/extensions", "extensions"]
 COPY --from=front-end ["/app/ui-build/build", "site"]
 
+# gin-gonic runs in debug mode without this
+ENV GIN_MODE=release
+
 # this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
-ENTRYPOINT ["/app/mizuagent"]
+ENTRYPOINT "/app/mizuagent"

Makefile

@@ -8,7 +8,7 @@ SHELL=/bin/bash
 # HELP
 # This will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help ui agent agent-debug cli tap docker
+.PHONY: help ui extensions extensions-debug agent agent-debug cli tap docker
 
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -19,7 +19,7 @@ help: ## This help.
 TS_SUFFIX="$(shell date '+%s')"
 GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
 BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
-export VER?=0.0
+export SEM_VER?=0.0.0
 
 ui: ## Build UI.
 	@(cd ui; npm i ; npm run build; )
@@ -31,29 +31,38 @@ cli: ## Build CLI.
 cli-debug: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build-debug
 
+build-cli-ci: ## Build CLI for CI.
+	@echo "building cli for ci"; cd cli && $(MAKE) build GIT_BRANCH=ci SUFFIX=ci
+
 agent: ## Build agent.
 	@(echo "building mizu agent .." )
 	@(cd agent; go build -o build/mizuagent main.go)
+	${MAKE} extensions
 	@ls -l agent/build
 
 agent-debug: ## Build agent for debug.
 	@(echo "building mizu agent for debug.." )
 	@(cd agent; go build -gcflags="all=-N -l" -o build/mizuagent main.go)
+	${MAKE} extensions-debug
 	@ls -l agent/build
 
 docker: ## Build and publish agent docker image.
 	$(MAKE) push-docker
 
-agent-docker: ## Build agent docker image.
-	@echo "Building agent docker image"
-	@docker build -t up9inc/mizu:devlatest .
-
 push: push-docker push-cli ## Build and publish agent docker image & CLI.
 
 push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
 	devops/build-push-featurebranch.sh
 
+push-docker-debug:
+	@echo "publishing debug Docker image .. "
+	devops/build-push-featurebranch-debug.sh
+
+build-docker-ci: ## Build agent docker image for CI.
+	@echo "building docker image for ci"
+	devops/build-agent-ci.sh
+
 push-cli: ## Build and publish CLI.
 	@echo "publishing CLI .. "
 	@cd cli; $(MAKE) build-all
@@ -62,7 +71,7 @@ push-cli: ## Build and publish CLI.
 	gsutil cp -r ./cli/bin/* gs://${BUCKET_PATH}/
 	gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
 
-clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
+clean: clean-ui clean-agent clean-cli clean-docker clean-extensions ## Clean all build artifacts.
 
 clean-ui: ## Clean UI.
 	@(rm -rf ui/build ; echo "UI cleanup done" )
@@ -73,32 +82,23 @@ clean-agent: ## Clean agent.
 clean-cli:  ## Clean CLI.
 	@(cd cli; make clean ; echo "CLI cleanup done" )
 
-clean-docker:  ## Run clean docker
+clean-extensions:  ## Clean extensions
+	@(rm -rf tap/extensions/*.so ; echo "Extensions cleanup done" )
+
+clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
-test-lint:  ## Run lint on all modules
-	cd agent && golangci-lint run
-	cd shared && golangci-lint run
-	cd tap && golangci-lint run
-	cd cli && golangci-lint run
-	cd acceptanceTests && golangci-lint run
-	cd tap/api && golangci-lint run
-	cd tap/extensions/ && for D in */; do cd $$D && golangci-lint run && cd ..; done
+extensions-debug:
+	devops/build_extensions_debug.sh
 
-test-cli:  ## Run cli tests
+extensions:
+	devops/build_extensions.sh
+
+test-cli:
 	@echo "running cli tests"; cd cli && $(MAKE) test
 
-test-agent:  ## Run agent tests
+test-agent:
 	@echo "running agent tests"; cd agent && $(MAKE) test
 
-test-shared:  ## Run shared tests
-	@echo "running shared tests"; cd shared && $(MAKE) test
-
-test-extensions:  ## Run extensions tests
-	@echo "running http tests"; cd tap/extensions/http && $(MAKE) test
-	@echo "running redis tests"; cd tap/extensions/redis && $(MAKE) test
-	@echo "running kafka tests"; cd tap/extensions/kafka && $(MAKE) test
-	@echo "running amqp tests"; cd tap/extensions/amqp && $(MAKE) test
-
-acceptance-test:  ## Run acceptance tests
+acceptance-test:
 	@echo "running acceptance tests"; cd acceptanceTests && $(MAKE) test

README.md

@@ -1,17 +1,11 @@
 ![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
 
 <p align="center">
-    <a href="https://github.com/up9inc/mizu/blob/main/LICENSE">
-        <img alt="GitHub License" src="https://img.shields.io/github/license/up9inc/mizu?logo=GitHub&style=flat-square">
-    </a>
     <a href="https://github.com/up9inc/mizu/releases/latest">
         <img alt="GitHub Latest Release" src="https://img.shields.io/github/v/release/up9inc/mizu?logo=GitHub&style=flat-square">
     </a>
-    <a href="https://hub.docker.com/r/up9inc/mizu">
-      <img alt="Docker pulls" src="https://img.shields.io/docker/pulls/up9inc/mizu?color=%23099cec&logo=Docker&style=flat-square">
-    </a>
-    <a href="https://hub.docker.com/r/up9inc/mizu">
-      <img alt="Image size" src="https://img.shields.io/docker/image-size/up9inc/mizu/latest?logo=Docker&style=flat-square">
+    <a href="https://github.com/up9inc/mizu/blob/main/LICENSE">
+        <img alt="GitHub License" src="https://img.shields.io/github/license/up9inc/mizu?logo=GitHub&style=flat-square">
     </a>
     <a href="https://join.slack.com/t/up9/shared_invite/zt-tfjnduli-QzlR8VV4Z1w3YnPIAJfhlQ">
       <img alt="Slack" src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social">
@@ -26,17 +20,178 @@ Think TCPDump and Wireshark re-invented for Kubernetes.
 
 ![Simple UI](assets/mizu-ui.png)
 
-## Quickstart and documentation
+## Features
 
-You can run Mizu on any Kubernetes cluster (version of 1.16.0 or higher) in a matter of seconds. See the [Mizu Getting Started Guide](https://getmizu.io/docs/) for how.
+- Simple and powerful CLI
+- Monitoring network traffic in real-time. Supported protocols:
+  - [HTTP/1.x](https://datatracker.ietf.org/doc/html/rfc2616) (REST, GraphQL, SOAP, etc.)
+  - [HTTP/2](https://datatracker.ietf.org/doc/html/rfc7540) (gRPC)
+  - [AMQP](https://www.rabbitmq.com/amqp-0-9-1-reference.html) (RabbitMQ, Apache Qpid, etc.)
+  - [Apache Kafka](https://kafka.apache.org/protocol)
+  - [Redis](https://redis.io/topics/protocol)
+- Works with Kubernetes APIs. No installation or code instrumentation
+- Rich filtering
 
-For more comprehensive documentation, start with the [docs](https://getmizu.io/docs/mizu/mizu-cli).
+## Requirements
 
-## Working in this repo
+A Kubernetes server version of 1.16.0 or higher is required.
 
-We ❤️ pull requests! See [CONTRIBUTING.md](docs/CONTRIBUTING.md) for info on contributing changes. <br />
-In the wiki you can find an intorduction to [mizu components](https://github.com/up9inc/mizu/wiki/Introduction-to-Mizu), and [development workflows](https://github.com/up9inc/mizu/wiki/Development-Workflows).
+## Download
 
-## Code of Conduct
+Download Mizu for your platform and operating system
 
-This project is for everyone. We ask that our users and contributors take a few minutes to review our [Code of Conduct](docs/CODE_OF_CONDUCT.md).
+### Latest Stable Release
+
+* for MacOS - Intel 
+```
+curl -Lo mizu \
+https://github.com/up9inc/mizu/releases/latest/download/mizu_darwin_amd64 \
+&& chmod 755 mizu
+```
+ 
+* for Linux - Intel 64bit
+```
+curl -Lo mizu \
+https://github.com/up9inc/mizu/releases/latest/download/mizu_linux_amd64 \
+&& chmod 755 mizu
+``` 
+
+SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page
+
+### Development (unstable) Build
+Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page
+
+## How to Run
+
+1. Find pods you'd like to tap to in your Kubernetes cluster
+2. Run `mizu tap` or `mizu tap PODNAME`
+3. Open browser on `http://localhost:8899` **or** as instructed in the CLI
+4. Watch the API traffic flowing
+5. Type ^C to stop
+
+## Examples
+
+Run `mizu help` for usage options
+
+To tap all pods in current namespace - 
+``` 
+ $ kubectl get pods 
+ NAME                            READY   STATUS    RESTARTS   AGE
+ carts-66c77f5fbb-fq65r          2/2     Running   0          20m
+ catalogue-5f4cb7cf5-7zrmn       2/2     Running   0          20m
+ front-end-649fc5fd6-kqbtn       2/2     Running   0          20m
+ ..
+
+ $ mizu tap
+ +carts-66c77f5fbb-fq65r
+ +catalogue-5f4cb7cf5-7zrmn
+ +front-end-649fc5fd6-kqbtn
+ Web interface is now available at http://localhost:8899
+ ^C
+```
+
+
+### To tap specific pod
+```bash
+ $ kubectl get pods 
+ NAME                            READY   STATUS    RESTARTS   AGE
+ front-end-649fc5fd6-kqbtn       2/2     Running   0          7m
+ ..
+
+ $ mizu tap front-end-649fc5fd6-kqbtn
+ +front-end-649fc5fd6-kqbtn
+ Web interface is now available at http://localhost:8899
+ ^C
+```
+
+### To tap multiple pods using regex
+```bash
+ $ kubectl get pods 
+ NAME                            READY   STATUS    RESTARTS   AGE
+ carts-66c77f5fbb-fq65r          2/2     Running   0          20m
+ catalogue-5f4cb7cf5-7zrmn       2/2     Running   0          20m
+ front-end-649fc5fd6-kqbtn       2/2     Running   0          20m
+ ..
+
+ $ mizu tap "^ca.*"
+ +carts-66c77f5fbb-fq65r
+ +catalogue-5f4cb7cf5-7zrmn
+ Web interface is now available at http://localhost:8899
+ ^C
+```
+
+## Configuration
+
+Mizu can optionally work with a config file that can be provided as a CLI argument (using `--set config-path=<PATH>`) or if not provided, will be stored at ${HOME}/.mizu/config.yaml 
+In case of partial configuration defined, all other fields will be used with defaults <br />
+You can always override the defaults or config file with CLI flags
+
+To get the default config params run `mizu config` <br />
+To generate a new config file with default values use `mizu config -r`
+
+
+## Advanced Usage
+
+### Kubeconfig
+
+It is possible to change the kubeconfig path using `KUBECONFIG` environment variable or the command like flag
+with `--set kube-config-path=<PATH>`. </br >
+If both are not set - Mizu assumes that configuration is at `${HOME}/.kube/config`
+
+### Namespace-Restricted Mode
+
+Some users have permission to only manage resources in one particular namespace assigned to them
+By default `mizu tap` creates a new namespace `mizu` for all of its Kubernetes resources. In order to instead install
+Mizu in an existing namespace, set the `mizu-resources-namespace` config option
+
+If `mizu-resources-namespace` is set to a value other than the default `mizu`, Mizu will operate in a
+Namespace-Restricted mode. It will only tap pods in `mizu-resources-namespace`. This way Mizu only requires permissions
+to the namespace set by `mizu-resources-namespace`. The user must set the tapped namespace to the same namespace by
+using the `--namespace` flag or by setting `tap.namespaces` in the config file
+
+Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior
+
+For detailed list of k8s permissions see [PERMISSIONS](docs/PERMISSIONS.md) document
+
+### User agent filtering
+
+User-agent filtering (like health checks) - can be configured using command-line options:
+
+```shell
+$ mizu tap "^ca.*" --set tap.ignored-user-agents=kube-probe --set tap.ignored-user-agents=prometheus
++carts-66c77f5fbb-fq65r
++catalogue-5f4cb7cf5-7zrmn
+Web interface is now available at http://localhost:8899
+^C
+
+```
+Any request that contains `User-Agent` header with one of the specified values (`kube-probe` or `prometheus`) will not be captured
+
+### Traffic validation rules
+
+This feature allows you to define set of simple rules, and test the traffic against them.
+Such validation may test response for specific JSON fields, headers, etc.
+
+Please see [TRAFFIC RULES](docs/POLICY_RULES.md) page for more details and syntax.
+
+### OpenAPI Specification (OAS) Contract Monitoring
+
+An OAS/Swagger file can contain schemas under `parameters` and `responses` fields. With `--contract catalogue.yaml`
+CLI option, you can pass your API description to Mizu and the traffic will automatically be validated
+against the contracts.
+
+Please see [CONTRACT MONITORING](docs/CONTRACT_MONITORING.md) page for more details and syntax.
+
+### Configure proxy host 
+
+By default, mizu will be accessible via local host: 'http://localhost:8899', it is possible to change the host, for
+instance, to '0.0.0.0' which can grant access via machine IP address. This setting can be changed via command line
+flag `--set tap.proxy-host=<value>` or via config file:
+tap proxy-host: 0.0.0.0 and when changed it will support accessing by IP
+
+### Install Mizu standalone
+
+Mizu can be run detached from the cli using the install command: `mizu install`. This type of mizu instance will run
+indefinitely in the cluster.
+
+For more information please refer to [INSTALL STANDALONE](docs/INSTALL_STANDALONE.md)

acceptanceTests/.snyk

@@ -1,6 +0,0 @@
-# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
-version: v1.14.0
-ignore:
-  SNYK-GOLANG-GITHUBCOMPKGSFTP-569475:
-    - '*':
-      reason: None Given

acceptanceTests/config_test.go

@@ -23,19 +23,19 @@ func TestConfigRegenerate(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := GetCliPath()
+	cliPath, cliPathErr := getCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	configPath, configPathErr := GetConfigPath()
+	configPath, configPathErr := getConfigPath()
 	if configPathErr != nil {
 		t.Errorf("failed to get config path, err: %v", cliPathErr)
 		return
 	}
 
-	configCmdArgs := GetDefaultConfigCommandArgs()
+	configCmdArgs := getDefaultConfigCommandArgs()
 
 	configCmdArgs = append(configCmdArgs, "-r")
 
@@ -74,13 +74,13 @@ func TestConfigGuiPort(t *testing.T) {
 
 	for _, guiPort := range tests {
 		t.Run(fmt.Sprintf("%d", guiPort), func(t *testing.T) {
-			cliPath, cliPathErr := GetCliPath()
+			cliPath, cliPathErr := getCliPath()
 			if cliPathErr != nil {
 				t.Errorf("failed to get cli path, err: %v", cliPathErr)
 				return
 			}
 
-			configPath, configPathErr := GetConfigPath()
+			configPath, configPathErr := getConfigPath()
 			if configPathErr != nil {
 				t.Errorf("failed to get config path, err: %v", cliPathErr)
 				return
@@ -100,16 +100,16 @@ func TestConfigGuiPort(t *testing.T) {
 				return
 			}
 
-			tapCmdArgs := GetDefaultTapCommandArgs()
+			tapCmdArgs := getDefaultTapCommandArgs()
 
-			tapNamespace := GetDefaultTapNamespace()
+			tapNamespace := getDefaultTapNamespace()
 			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 			tapCmd := exec.Command(cliPath, tapCmdArgs...)
 			t.Logf("running command: %v", tapCmd.String())
 
 			t.Cleanup(func() {
-				if err := CleanupCommand(tapCmd); err != nil {
+				if err := cleanupCommand(tapCmd); err != nil {
 					t.Logf("failed to cleanup tap command, err: %v", err)
 				}
 
@@ -123,9 +123,9 @@ func TestConfigGuiPort(t *testing.T) {
 				return
 			}
 
-			apiServerUrl := GetApiServerUrl(guiPort)
+			apiServerUrl := getApiServerUrl(guiPort)
 
-			if err := WaitTapPodsReady(apiServerUrl); err != nil {
+			if err := waitTapPodsReady(apiServerUrl); err != nil {
 				t.Errorf("failed to start tap pods on time, err: %v", err)
 				return
 			}
@@ -147,13 +147,13 @@ func TestConfigSetGuiPort(t *testing.T) {
 
 	for _, guiPortStruct := range tests {
 		t.Run(fmt.Sprintf("%d", guiPortStruct.SetGuiPort), func(t *testing.T) {
-			cliPath, cliPathErr := GetCliPath()
+			cliPath, cliPathErr := getCliPath()
 			if cliPathErr != nil {
 				t.Errorf("failed to get cli path, err: %v", cliPathErr)
 				return
 			}
 
-			configPath, configPathErr := GetConfigPath()
+			configPath, configPathErr := getConfigPath()
 			if configPathErr != nil {
 				t.Errorf("failed to get config path, err: %v", cliPathErr)
 				return
@@ -173,9 +173,9 @@ func TestConfigSetGuiPort(t *testing.T) {
 				return
 			}
 
-			tapCmdArgs := GetDefaultTapCommandArgs()
+			tapCmdArgs := getDefaultTapCommandArgs()
 
-			tapNamespace := GetDefaultTapNamespace()
+			tapNamespace := getDefaultTapNamespace()
 			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 			tapCmdArgs = append(tapCmdArgs, "--set", fmt.Sprintf("tap.gui-port=%v", guiPortStruct.SetGuiPort))
@@ -184,7 +184,7 @@ func TestConfigSetGuiPort(t *testing.T) {
 			t.Logf("running command: %v", tapCmd.String())
 
 			t.Cleanup(func() {
-				if err := CleanupCommand(tapCmd); err != nil {
+				if err := cleanupCommand(tapCmd); err != nil {
 					t.Logf("failed to cleanup tap command, err: %v", err)
 				}
 
@@ -198,9 +198,9 @@ func TestConfigSetGuiPort(t *testing.T) {
 				return
 			}
 
-			apiServerUrl := GetApiServerUrl(guiPortStruct.SetGuiPort)
+			apiServerUrl := getApiServerUrl(guiPortStruct.SetGuiPort)
 
-			if err := WaitTapPodsReady(apiServerUrl); err != nil {
+			if err := waitTapPodsReady(apiServerUrl); err != nil {
 				t.Errorf("failed to start tap pods on time, err: %v", err)
 				return
 			}
@@ -222,13 +222,13 @@ func TestConfigFlagGuiPort(t *testing.T) {
 
 	for _, guiPortStruct := range tests {
 		t.Run(fmt.Sprintf("%d", guiPortStruct.FlagGuiPort), func(t *testing.T) {
-			cliPath, cliPathErr := GetCliPath()
+			cliPath, cliPathErr := getCliPath()
 			if cliPathErr != nil {
 				t.Errorf("failed to get cli path, err: %v", cliPathErr)
 				return
 			}
 
-			configPath, configPathErr := GetConfigPath()
+			configPath, configPathErr := getConfigPath()
 			if configPathErr != nil {
 				t.Errorf("failed to get config path, err: %v", cliPathErr)
 				return
@@ -248,9 +248,9 @@ func TestConfigFlagGuiPort(t *testing.T) {
 				return
 			}
 
-			tapCmdArgs := GetDefaultTapCommandArgs()
+			tapCmdArgs := getDefaultTapCommandArgs()
 
-			tapNamespace := GetDefaultTapNamespace()
+			tapNamespace := getDefaultTapNamespace()
 			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 			tapCmdArgs = append(tapCmdArgs, "-p", fmt.Sprintf("%v", guiPortStruct.FlagGuiPort))
@@ -259,7 +259,7 @@ func TestConfigFlagGuiPort(t *testing.T) {
 			t.Logf("running command: %v", tapCmd.String())
 
 			t.Cleanup(func() {
-				if err := CleanupCommand(tapCmd); err != nil {
+				if err := cleanupCommand(tapCmd); err != nil {
 					t.Logf("failed to cleanup tap command, err: %v", err)
 				}
 
@@ -273,9 +273,9 @@ func TestConfigFlagGuiPort(t *testing.T) {
 				return
 			}
 
-			apiServerUrl := GetApiServerUrl(guiPortStruct.FlagGuiPort)
+			apiServerUrl := getApiServerUrl(guiPortStruct.FlagGuiPort)
 
-			if err := WaitTapPodsReady(apiServerUrl); err != nil {
+			if err := waitTapPodsReady(apiServerUrl); err != nil {
 				t.Errorf("failed to start tap pods on time, err: %v", err)
 				return
 			}

acceptanceTests/create_user.sh

@@ -1,37 +0,0 @@
-#!/bin/bash
-
-# Create a user in Minikube cluster "minikube"
-# Create context for user
-# Usage:
-#  ./create_user.sh <username>
-
-set -e
-
-NEW_USERNAME=$1
-CERT_DIR="${HOME}/certs"
-KEY_FILE="${CERT_DIR}/${NEW_USERNAME}.key"
-CRT_FILE="${CERT_DIR}/${NEW_USERNAME}.crt"
-MINIKUBE_KEY_FILE="${HOME}/.minikube/ca.key"
-MINIKUBE_CRT_FILE="${HOME}/.minikube/ca.crt"
-DAYS=1
-
-echo "Creating user and context for username \"${NEW_USERNAME}\" in Minikube cluster"
-
-if ! command -v openssl &> /dev/null
-then
-	 echo "Installing openssl"
-	 sudo apt-get update
-	 sudo apt-get install openssl
-fi
-
-echo "Creating certificate for user \"${NEW_USERNAME}\""
-mkdir -p ${CERT_DIR}
-echo "Generating key \"${KEY_FILE}\""
-openssl genrsa -out "${KEY_FILE}" 2048
-echo "Generating crt \"${CRT_FILE}\""
-openssl req -new -key "${KEY_FILE}" -out "${CRT_FILE}" -subj "/CN=${NEW_USERNAME}/O=group1"
-openssl x509 -req -in "${CRT_FILE}" -CA "${MINIKUBE_CRT_FILE}" -CAkey "${MINIKUBE_KEY_FILE}" -CAcreateserial -out "${CRT_FILE}" -days $DAYS
-
-echo "Creating context for user \"${NEW_USERNAME}\""
-kubectl config set-credentials "${NEW_USERNAME}" --client-certificate="${CRT_FILE}" --client-key="${KEY_FILE}"
-kubectl config set-context "${NEW_USERNAME}" --cluster=minikube --user="${NEW_USERNAME}"

acceptanceTests/cypress/integration/configurations/Default.json

@@ -4,31 +4,19 @@
   "viewportHeight": 1080,
   "video": false,
   "screenshotOnRunFailure": false,
-  "defaultCommandTimeout": 6000,
   "testFiles": [
     "tests/GuiPort.js",
     "tests/MultipleNamespaces.js",
     "tests/Redact.js",
     "tests/NoRedact.js",
     "tests/Regex.js",
-    "tests/RegexMasking.js",
-    "tests/IgnoredUserAgents.js",
-    "tests/UiTest.js",
-    "tests/Redis.js",
-    "tests/Rabbit.js",
-    "tests/serviceMapFunction.js"
+    "tests/RegexMasking.js"
   ],
 
   "env": {
     "testUrl": "http://localhost:8899/",
     "redactHeaderContent": "User-Header[REDACTED]",
     "redactBodyContent": "{ \"User\": \"[REDACTED]\" }",
-    "regexMaskingBodyContent": "[REDACTED]",
-    "greenFilterColor": "rgb(210, 250, 210)",
-    "redFilterColor": "rgb(250, 214, 220)",
-    "bodyJsonClass": ".hljs",
-    "mizuWidth": 1920,
-    "normalMizuHeight": 1080,
-    "hugeMizuHeight": 3500
+    "regexMaskingBodyContent": "[REDACTED]"
   }
 }

acceptanceTests/cypress/integration/configurations/HugeMizu.json

@@ -0,0 +1,14 @@
+{
+  "watchForFileChanges":false,
+  "viewportWidth": 1920,
+  "viewportHeight": 3500,
+  "video": false,
+  "screenshotOnRunFailure": false,
+  "testFiles": [
+    "tests/IgnoredUserAgents.js"
+  ],
+
+  "env": {
+    "testUrl": "http://localhost:8899/"
+  }
+}

acceptanceTests/cypress/integration/testHelpers/StatusBarHelper.js

@@ -1,7 +1,8 @@
 const columns = {podName : 1, namespace : 2, tapping : 3};
+const greenStatusImageSrc = '/static/media/success.662997eb.svg';
 
 function getDomPathInStatusBar(line, column) {
-    return `[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) > :nth-child(${line}) > :nth-child(${column})`;
+    return `.expandedStatusBar > :nth-child(2) > > :nth-child(2) > :nth-child(${line}) > :nth-child(${column})`;
 }
 
 export function checkLine(line, expectedValues) {
@@ -11,14 +12,14 @@ export function checkLine(line, expectedValues) {
 
         cy.get(getDomPathInStatusBar(line, columns.namespace)).invoke('text').then(namespaceValue => {
             expect(namespaceValue).to.equal(expectedValues.namespace);
-            cy.get(getDomPathInStatusBar(line, columns.tapping)).children().should('have.attr', 'src').and("match", /success.*\.svg/);
+            cy.get(getDomPathInStatusBar(line, columns.tapping)).children().should('have.attr', 'src', greenStatusImageSrc);
         });
     });
 }
 
 export function findLineAndCheck(expectedValues) {
-    cy.get('[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) > > :nth-child(1)').then(pods => {
-        cy.get('[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) > > :nth-child(2)').then(namespaces => {
+    cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) > > :nth-child(1)').then(pods => {
+        cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) > > :nth-child(2)').then(namespaces => {
             // organizing namespaces array
             const podObjectsArray = Object.values(pods ?? {});
             const namespacesObjectsArray = Object.values(namespaces ?? {});

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -1,11 +1,3 @@
-export const valueTabs = {
-    response: 'RESPONSE',
-    request: 'REQUEST',
-    none: null
-}
-
-const maxEntriesInDom = 13;
-
 export function isValueExistsInElement(shouldInclude, content, domPathToContainer){
     it(`should ${shouldInclude ? '' : 'not'} include '${content}'`, function () {
         cy.get(domPathToContainer).then(htmlText => {
@@ -15,175 +7,3 @@ export function isValueExistsInElement(shouldInclude, content, domPathToContaine
         });
     });
 }
-
-export function resizeToHugeMizu() {
-    cy.viewport(Cypress.env('mizuWidth'), Cypress.env('hugeMizuHeight'));
-}
-
-export function resizeToNormalMizu() {
-    cy.viewport(Cypress.env('mizuWidth'), Cypress.env('normalMizuHeight'));
-}
-
-export function verifyMinimumEntries() {
-    const entriesSent = Cypress.env('entriesCount');
-    const minimumEntries = Math.round((0.75 * entriesSent));
-
-    it(`Making sure that mizu shows at least ${minimumEntries} entries`, function () {
-        cy.get('#total-entries').then(number => {
-            const getNum = () => {
-                return parseInt(number.text());
-            };
-
-            cy.wrap({num: getNum}).invoke('num').should('be.gt', minimumEntries);
-        });
-    });
-}
-
-export function leftTextCheck(entryNum, path, expectedText) {
-    cy.get(`#list #entry-${entryNum} ${path}`).invoke('text').should('eq', expectedText);
-}
-
-export function leftOnHoverCheck(entryNum, path, filterName) {
-    cy.get(`#list #entry-${entryNum} ${path}`).trigger('mouseover');
-    cy.get(`#list #entry-${entryNum} [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(filterName));
-}
-
-export function rightTextCheck(path, expectedText) {
-    cy.get(`#rightSideContainer ${path}`).should('have.text', expectedText);
-}
-
-export function rightOnHoverCheck(path, expectedText) {
-    cy.get(`#rightSideContainer ${path}`).trigger('mouseover');
-    cy.get(`#rightSideContainer [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(expectedText));
-}
-
-export function checkThatAllEntriesShown() {
-    cy.get('#entries-length').then(number => {
-        if (number.text() === '1')
-            cy.get('[title="Fetch old records"]').click();
-    });
-}
-
-export function checkFilterByMethod(funcDict) {
-    const {protocol, method, methodQuery, summary, summaryQuery} = funcDict;
-    const summaryDict = getSummaryDict(summary, summaryQuery);
-    const methodDict = getMethodDict(method, methodQuery);
-    const protocolDict = getProtocolDict(protocol.name, protocol.text);
-
-    it(`Testing the method: ${method}`, function () {
-        // applying filter
-        cy.get('.w-tc-editor-text').clear().type(methodQuery);
-        cy.get('[type="submit"]').click();
-        cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
-
-        cy.get('#entries-length').then(number => {
-            // if the entries list isn't expanded it expands here
-            if (number.text() === '0' || number.text() === '1') // todo change when TRA-4262 is fixed
-                cy.get('[title="Fetch old records"]').click();
-
-            cy.get('#entries-length').should('not.have.text', '0').and('not.have.text', '1').then(() => {
-                cy.get(`#list [id]`).then(elements => {
-                    const listElmWithIdAttr = Object.values(elements);
-                    let doneCheckOnFirst = false;
-
-                    cy.get('#entries-length').invoke('text').then(len => {
-                        resizeIfNeeded(len);
-                        listElmWithIdAttr.forEach(entry => {
-                            if (entry?.id && entry.id.match(RegExp(/entry-(\d{2}|\d{1})$/gm))) {
-                                const entryNum = getEntryNumById(entry.id);
-
-                                leftTextCheck(entryNum, methodDict.pathLeft, methodDict.expectedText);
-                                leftTextCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedTextLeft);
-                                if (summaryDict)
-                                    leftTextCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedText);
-
-                                if (!doneCheckOnFirst) {
-                                    deepCheck(funcDict, protocolDict, methodDict, entry);
-                                    doneCheckOnFirst = true;
-                                }
-                            }
-                        });
-                        resizeIfNeeded(len);
-                    });
-                });
-            });
-        });
-    });
-}
-
-function resizeIfNeeded(entriesLen) {
-    if (entriesLen > maxEntriesInDom){
-        Cypress.config().viewportHeight === Cypress.env('normalMizuHeight') ?
-            resizeToHugeMizu() : resizeToNormalMizu()
-    }
-}
-
-function deepCheck(generalDict, protocolDict, methodDict, entry) {
-    const entryNum = getEntryNumById(entry.id);
-    const {summary, value} = generalDict;
-    const summaryDict = getSummaryDict(summary);
-
-    leftOnHoverCheck(entryNum, methodDict.pathLeft, methodDict.expectedOnHover);
-    leftOnHoverCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedOnHover);
-    if (summaryDict)
-        leftOnHoverCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedOnHover);
-
-    cy.get(`#${entry.id}`).click();
-
-    rightTextCheck(methodDict.pathRight, methodDict.expectedText);
-    rightTextCheck(protocolDict.pathRight, protocolDict.expectedTextRight);
-    if (summaryDict)
-        rightTextCheck(summaryDict.pathRight, summaryDict.expectedText);
-
-    rightOnHoverCheck(methodDict.pathRight, methodDict.expectedOnHover);
-    rightOnHoverCheck(protocolDict.pathRight, protocolDict.expectedOnHover);
-    if (summaryDict)
-        rightOnHoverCheck(summaryDict.pathRight, summaryDict.expectedOnHover);
-
-    if (value) {
-        if (value.tab === valueTabs.response)
-            // temporary fix, change to some "data-cy" attribute,
-            // this will fix the issue that happen because we have "response:" in the header of the right side
-            cy.get('#rightSideContainer > :nth-child(3)').contains('Response').click();
-        cy.get(Cypress.env('bodyJsonClass')).then(text => {
-            expect(text.text()).to.match(value.regex)
-        });
-    }
-}
-
-function getSummaryDict(value, query) {
-    if (value) {
-        return {
-            pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
-            pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
-            expectedText: value,
-            expectedOnHover: query
-        };
-    }
-    else {
-        return null;
-    }
-}
-
-function getMethodDict(value, query) {
-    return {
-        pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
-        pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
-        expectedText: value,
-        expectedOnHover: query
-    };
-}
-
-function getProtocolDict(protocol, protocolText) {
-    return {
-        pathLeft: '> :nth-child(1) > :nth-child(1)',
-        pathRight: '> :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(1)',
-        expectedTextLeft: protocol.toUpperCase(),
-        expectedTextRight: protocolText,
-        expectedOnHover: protocol.toLowerCase()
-    };
-}
-
-function getEntryNumById (id) {
-    return parseInt(id.split('-')[1]);
-}

acceptanceTests/cypress/integration/tests/GuiPort.js

@@ -1,13 +1,8 @@
-import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
-
 it('check', function () {
-    const podName = Cypress.env('name'), namespace = Cypress.env('namespace');
-    const port = Cypress.env('port');
-    cy.intercept('GET', `http://localhost:${port}/status/tap`).as('statusTap');
+    cy.visit(`http://localhost:${Cypress.env('port')}/`);
 
-    cy.visit(`http://localhost:${port}`);
-    cy.wait('@statusTap').its('response.statusCode').should('match', /^2\d{2}/);
-
-    cy.get(`[data-cy="expandedStatusBar"]`).trigger('mouseover',{force: true});
-    findLineAndCheck(getExpectedDetailsDict(podName, namespace));
+    cy.get('.header').should('be.visible');
+    cy.get('.TrafficPageHeader').should('be.visible');
+    cy.get('.TrafficPage-ListContainer').should('be.visible');
+    cy.get('.TrafficPage-Container').should('be.visible');
 });

acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js

@@ -1,24 +1,28 @@
-import {
-    checkThatAllEntriesShown,
-    isValueExistsInElement,
-    resizeToHugeMizu,
-} from "../testHelpers/TrafficHelper";
+import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
 });
 
-checkEntries();
+it('going through each entry', function () {
+    cy.get('#total-entries').then(number => {
+        const getNum = () => {
+            const numOfEntries = number.text();
+            return parseInt(numOfEntries);
+        };
+        cy.wrap({ there: getNum }).invoke('there').should('be.gte', 25);
 
-function checkEntries() {
-    it('checking all entries', function () {
         checkThatAllEntriesShown();
-        resizeToHugeMizu();
 
-        cy.get('#total-entries').then(number => {
-            const numOfEntries = parseInt(number.text());
-            [...Array(numOfEntries).keys()].map(checkEntry);
-        });
+        const entriesNum = getNum();
+        [...Array(entriesNum).keys()].map(checkEntry);
+    });
+});
+
+function checkThatAllEntriesShown() {
+    cy.get('#entries-length').then(number => {
+        if (number.text() === '1')
+            cy.get('[title="Fetch old records"]').click();
     });
 }
 

acceptanceTests/cypress/integration/tests/MultipleNamespaces.js

@@ -2,7 +2,7 @@ import {findLineAndCheck, getExpectedDetailsDict} from '../testHelpers/StatusBar
 
 it('opening', function () {
     cy.visit(Cypress.env('testUrl'));
-    cy.get(`[data-cy="podsCountText"]`).trigger('mouseover');
+    cy.get('.podsCount').trigger('mouseover');
 });
 
 [1, 2, 3].map(doItFunc);

acceptanceTests/cypress/integration/tests/NoRedact.js

@@ -2,7 +2,7 @@ import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
-});
+})
 
 isValueExistsInElement(false, Cypress.env('redactHeaderContent'), '#tbody-Headers');
-isValueExistsInElement(false, Cypress.env('redactBodyContent'), Cypress.env('bodyJsonClass'));
+isValueExistsInElement(false, Cypress.env('redactBodyContent'), '.hljs');

acceptanceTests/cypress/integration/tests/Rabbit.js

@@ -1,61 +0,0 @@
-import {checkFilterByMethod, valueTabs,} from "../testHelpers/TrafficHelper";
-
-it('opening mizu', function () {
-    cy.visit(Cypress.env('testUrl'));
-});
-
-const rabbitProtocolDetails = {name: 'AMQP', text: 'Advanced Message Queuing Protocol 0-9-1'};
-
-checkFilterByMethod({
-    protocol: rabbitProtocolDetails,
-    method: 'exchange declare',
-    methodQuery: 'request.method == "exchange declare"',
-    summary: 'exchange',
-    summaryQuery: 'request.exchange == "exchange"',
-    value: null
-});
-
-checkFilterByMethod({
-    protocol: rabbitProtocolDetails,
-    method: 'queue declare',
-    methodQuery: 'request.method == "queue declare"',
-    summary: 'queue',
-    summaryQuery: 'request.queue == "queue"',
-    value: null
-});
-
-checkFilterByMethod({
-    protocol: rabbitProtocolDetails,
-    method:  'queue bind',
-    methodQuery: 'request.method == "queue bind"',
-    summary: 'queue',
-    summaryQuery: 'request.queue == "queue"',
-    value: null
-});
-
-checkFilterByMethod({
-    protocol: rabbitProtocolDetails,
-    method: 'basic publish',
-    methodQuery: 'request.method == "basic publish"',
-    summary: 'exchange',
-    summaryQuery: 'request.exchange == "exchange"',
-    value: {tab: valueTabs.request, regex: /^message$/mg}
-});
-
-checkFilterByMethod({
-    protocol: rabbitProtocolDetails,
-    method:  'basic consume',
-    methodQuery: 'request.method == "basic consume"',
-    summary: 'queue',
-    summaryQuery: 'request.queue == "queue"',
-    value: null
-});
-
-checkFilterByMethod({
-    protocol: rabbitProtocolDetails,
-    method:  'basic deliver',
-    methodQuery: 'request.method == "basic deliver"',
-    summary: 'exchange',
-    summaryQuery: 'request.queue == "exchange"',
-    value: {tab: valueTabs.request, regex: /^message$/mg}
-});

acceptanceTests/cypress/integration/tests/Redact.js

@@ -2,7 +2,7 @@ import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
-});
+})
 
 isValueExistsInElement(true, Cypress.env('redactHeaderContent'), '#tbody-Headers');
-isValueExistsInElement(true, Cypress.env('redactBodyContent'), Cypress.env('bodyJsonClass'));
+isValueExistsInElement(true, Cypress.env('redactBodyContent'), '.hljs');

acceptanceTests/cypress/integration/tests/Redis.js

@@ -1,52 +0,0 @@
-import {checkFilterByMethod, valueTabs,} from "../testHelpers/TrafficHelper";
-
-it('opening mizu', function () {
-    cy.visit(Cypress.env('testUrl'));
-});
-
-const redisProtocolDetails = {name: 'redis', text: 'Redis Serialization Protocol'};
-
-checkFilterByMethod({
-    protocol: redisProtocolDetails,
-    method: 'PING',
-    methodQuery: 'request.command == "PING"',
-    summary: null,
-    summaryQuery: '',
-    value: null
-})
-
-checkFilterByMethod({
-    protocol: redisProtocolDetails,
-    method: 'SET',
-    methodQuery: 'request.command == "SET"',
-    summary: 'key',
-    summaryQuery: 'request.key == "key"',
-    value: {tab: valueTabs.request, regex: /^\[value, keepttl]$/mg}
-})
-
-checkFilterByMethod({
-    protocol: redisProtocolDetails,
-    method: 'EXISTS',
-    methodQuery: 'request.command == "EXISTS"',
-    summary: 'key',
-    summaryQuery: 'request.key == "key"',
-    value: {tab: valueTabs.response, regex: /^1$/mg}
-})
-
-checkFilterByMethod({
-    protocol: redisProtocolDetails,
-    method: 'GET',
-    methodQuery: 'request.command == "GET"',
-    summary: 'key',
-    summaryQuery: 'request.key == "key"',
-    value: {tab: valueTabs.response, regex: /^value$/mg}
-})
-
-checkFilterByMethod({
-    protocol: redisProtocolDetails,
-    method: 'DEL',
-    methodQuery: 'request.command == "DEL"',
-    summary: 'key',
-    summaryQuery: 'request.key == "key"',
-    value: {tab: valueTabs.response, regex: /^1$|^0$/mg}
-})

acceptanceTests/cypress/integration/tests/Regex.js

@@ -3,9 +3,9 @@ import {getExpectedDetailsDict, checkLine} from '../testHelpers/StatusBarHelper'
 
 it('opening', function () {
     cy.visit(Cypress.env('testUrl'));
-    cy.get(`[data-cy="podsCountText"]`).trigger('mouseover');
+    cy.get('.podsCount').trigger('mouseover');
 
-    cy.get('[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) >').should('have.length', 1); // one line
+    cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) >').should('have.length', 1); // one line
 
     checkLine(1, getExpectedDetailsDict(Cypress.env('name'), Cypress.env('namespace')));
 });

acceptanceTests/cypress/integration/tests/RegexMasking.js

@@ -2,6 +2,6 @@ import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
-});
+})
 
-isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), Cypress.env('bodyJsonClass'));
+isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), '.hljs');

acceptanceTests/cypress/integration/tests/UiTest.js

@@ -1,366 +0,0 @@
-import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
-import {
-    leftOnHoverCheck,
-    leftTextCheck,
-    resizeToHugeMizu,
-    resizeToNormalMizu,
-    rightOnHoverCheck,
-    rightTextCheck,
-    verifyMinimumEntries
-} from "../testHelpers/TrafficHelper";
-
-const refreshWaitTimeout = 10000;
-
-
-const fullParam = Cypress.env('arrayDict'); // "Name:fooNamespace:barName:foo1Namespace:bar1"
-const podsArray = fullParam.split('Name:').slice(1); // ["fooNamespace:bar", "foo1Namespace:bar1"]
-podsArray.forEach((podStr, index) => {
-    const podAndNamespaceArr = podStr.split('Namespace:'); // [foo, bar] / [foo1, bar1]
-    podsArray[index] = getExpectedDetailsDict(podAndNamespaceArr[0], podAndNamespaceArr[1]);
-});
-
-it('opening mizu', function () {
-    cy.visit(Cypress.env('testUrl'));
-});
-
-verifyMinimumEntries();
-
-it('top bar check', function () {
-    cy.get(`[data-cy="podsCountText"]`).trigger('mouseover');
-    podsArray.map(findLineAndCheck);
-    cy.reload();
-});
-
-it('filtering guide check', function () {
-    cy.reload();
-    cy.get('[title="Open Filtering Guide (Cheatsheet)"]').click();
-    cy.get('#modal-modal-title').should('be.visible');
-    cy.get('[lang="en"]').click(0, 0);
-    cy.get('#modal-modal-title').should('not.exist');
-});
-
-it('right side sanity test', function () {
-    cy.get('#entryDetailedTitleElapsedTime').then(timeInMs => {
-        const time = timeInMs.text();
-        if (time < '0ms') {
-            throw new Error(`The time in the top line cannot be negative ${time}`);
-        }
-    });
-
-    // temporary fix, change to some "data-cy" attribute,
-    // this will fix the issue that happen because we have "response:" in the header of the right side
-    cy.get('#rightSideContainer > :nth-child(3)').contains('Response').click();
-
-    cy.get('#rightSideContainer [title="Status Code"]').then(status => {
-        const statusCode = status.text();
-        cy.contains('Status').parent().next().then(statusInDetails => {
-            const statusCodeInDetails = statusInDetails.text();
-
-            expect(statusCode).to.equal(statusCodeInDetails, 'The status code in the top line should match the status code in details');
-        });
-    });
-});
-
-checkIllegalFilter('invalid filter');
-
-checkFilter({
-    name: 'http',
-    leftSidePath: '> :nth-child(1) > :nth-child(1)',
-    leftSideExpectedText: 'HTTP',
-    rightSidePath: '[title=HTTP]',
-    rightSideExpectedText: 'Hypertext Transfer Protocol -- HTTP/1.1',
-    applyByEnter: true
-});
-
-checkFilter({
-    name: 'response.status == 200',
-    leftSidePath: '[title="Status Code"]',
-    leftSideExpectedText: '200',
-    rightSidePath: '> :nth-child(2) [title="Status Code"]',
-    rightSideExpectedText: '200',
-    applyByEnter: false
-});
-
-if (Cypress.env('shouldCheckSrcAndDest')) {
-    serviceMapCheck();
-
-    checkFilter({
-        name: 'src.name == ""',
-        leftSidePath: '[title="Source Name"]',
-        leftSideExpectedText: '[Unresolved]',
-        rightSidePath: '> :nth-child(2) [title="Source Name"]',
-        rightSideExpectedText: '[Unresolved]',
-        applyByEnter: false
-    });
-
-    checkFilter({
-        name: `dst.name == "httpbin.mizu-tests"`,
-        leftSidePath: '> :nth-child(3) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
-        leftSideExpectedText: 'httpbin.mizu-tests',
-        rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
-        rightSideExpectedText: 'httpbin.mizu-tests',
-        applyByEnter: false
-    });
-}
-
-checkFilter({
-    name: 'request.method == "GET"',
-    leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
-    leftSideExpectedText: 'GET',
-    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
-    rightSideExpectedText: 'GET',
-    applyByEnter: true
-});
-
-checkFilter({
-    name: 'request.path == "/get"',
-    leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
-    leftSideExpectedText: '/get',
-    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
-    rightSideExpectedText: '/get',
-    applyByEnter: false
-});
-
-checkFilter({
-    name: 'src.ip == "127.0.0.1"',
-    leftSidePath: '[title="Source IP"]',
-    leftSideExpectedText: '127.0.0.1',
-    rightSidePath: '> :nth-child(2) [title="Source IP"]',
-    rightSideExpectedText: '127.0.0.1',
-    applyByEnter: false
-});
-
-checkFilterNoResults('request.method == "POST"');
-
-function checkFilterNoResults(filterName) {
-    it(`checking the filter: ${filterName}. Expecting no results`, function () {
-        cy.get('#total-entries').then(number => {
-            const totalEntries = number.text();
-
-            // applying the filter
-            cy.get('.w-tc-editor-text').type(filterName);
-            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
-            cy.get('[type="submit"]').click();
-
-            // waiting for the entries number to load
-            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-
-            // the DOM should show 0 entries
-            cy.get('#entries-length').should('have.text', '0');
-
-            // going through every potential entry and verifies that it doesn't exist
-            [...Array(parseInt(totalEntries)).keys()].map(shouldNotExist);
-
-            cy.get('[title="Fetch old records"]').click();
-            cy.get('#noMoreDataTop', {timeout: refreshWaitTimeout}).should('be.visible');
-            cy.get('#entries-length').should('have.text', '0'); // after loading all entries there should still be 0 entries
-
-            // reloading then waiting for the entries number to load
-            cy.reload();
-            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-        });
-    });
-}
-
-function shouldNotExist(entryNum) {
-    cy.get(`entry-${entryNum}`).should('not.exist');
-}
-
-function checkIllegalFilter(illegalFilterName) {
-    it(`should show red search bar with the input: ${illegalFilterName}`, function () {
-        cy.reload();
-        cy.get('#total-entries').then(number => {
-            const totalEntries = number.text();
-
-            cy.get('.w-tc-editor-text').type(illegalFilterName);
-            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('redFilterColor'));
-            cy.get('[type="submit"]').click();
-
-            cy.get('[role="alert"]').should('be.visible');
-            cy.get('.w-tc-editor-text').clear();
-
-            // reloading then waiting for the entries number to load
-            cy.reload();
-            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-        });
-    });
-}
-function checkFilter(filterDetails){
-    const {name, leftSidePath, rightSidePath, rightSideExpectedText, leftSideExpectedText, applyByEnter} = filterDetails;
-    const entriesForDeeperCheck = 5;
-
-    it(`checking the filter: ${name}`, function () {
-        cy.get('#total-entries').should('not.have.text', '0').then(number => {
-            const totalEntries = number.text();
-
-            // checks the hover on the last entry (the only one in DOM at the beginning)
-            leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
-
-            cy.get('.w-tc-editor-text').clear();
-            // applying the filter with alt+enter or with the button
-            cy.get('.w-tc-editor-text').type(`${name}${applyByEnter ? '{alt+enter}' : ''}`);
-            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
-            if (!applyByEnter)
-                cy.get('[type="submit"]').click();
-
-            // only one entry in DOM after filtering, checking all checks on it
-            leftTextCheck(totalEntries - 1, leftSidePath, leftSideExpectedText);
-            leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
-            rightTextCheck(rightSidePath, rightSideExpectedText);
-            rightOnHoverCheck(rightSidePath, name);
-            checkRightSideResponseBody();
-
-            cy.get('[title="Fetch old records"]').click();
-            resizeToHugeMizu();
-
-            // waiting for the entries number to load
-            cy.get('#entries-length', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-
-            // checking only 'leftTextCheck' on all entries because the rest of the checks require more time
-            [...Array(parseInt(totalEntries)).keys()].forEach(entryNum => {
-                leftTextCheck(entryNum, leftSidePath, leftSideExpectedText);
-            });
-
-            // making the other 3 checks on the first X entries (longer time for each check)
-            deeperChcek(leftSidePath, rightSidePath, name, leftSideExpectedText, rightSideExpectedText, entriesForDeeperCheck);
-
-            // reloading then waiting for the entries number to load
-            resizeToNormalMizu();
-            cy.reload();
-            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-        });
-    });
-}
-
-function deeperChcek(leftSidePath, rightSidePath, filterName, leftSideExpectedText, rightSideExpectedText, entriesNumToCheck) {
-    [...Array(entriesNumToCheck).keys()].forEach(entryNum => {
-        leftOnHoverCheck(entryNum, leftSidePath, filterName);
-
-        cy.get(`#list #entry-${entryNum}`).click();
-        rightTextCheck(rightSidePath, rightSideExpectedText);
-        rightOnHoverCheck(rightSidePath, filterName);
-    });
-}
-
-function checkRightSideResponseBody() {
-    // temporary fix, change to some "data-cy" attribute,
-    // this will fix the issue that happen because we have "response:" in the header of the right side
-    cy.get('#rightSideContainer > :nth-child(3)').contains('Response').click();
-    clickCheckbox('Decode Base64');
-
-    cy.get(`${Cypress.env('bodyJsonClass')}`).then(value => {
-        const encodedBody = value.text();
-        const decodedBody = atob(encodedBody);
-        const responseBody = JSON.parse(decodedBody);
-
-        const expectdJsonBody = {
-            args: RegExp({}),
-            url: RegExp('http://.*/get'),
-            headers: {
-                "User-Agent": RegExp('[REDACTED]'),
-                "Accept-Encoding": RegExp('gzip'),
-                "X-Forwarded-Uri": RegExp('/api/v1/namespaces/.*/services/.*/proxy/get')
-            }
-        };
-
-        expect(responseBody.args).to.match(expectdJsonBody.args);
-        expect(responseBody.url).to.match(expectdJsonBody.url);
-        expect(responseBody.headers['User-Agent']).to.match(expectdJsonBody.headers['User-Agent']);
-        expect(responseBody.headers['Accept-Encoding']).to.match(expectdJsonBody.headers['Accept-Encoding']);
-        expect(responseBody.headers['X-Forwarded-Uri']).to.match(expectdJsonBody.headers['X-Forwarded-Uri']);
-
-        cy.get(`${Cypress.env('bodyJsonClass')}`).should('have.text', encodedBody);
-        clickCheckbox('Decode Base64');
-
-        cy.get(`${Cypress.env('bodyJsonClass')} > `).its('length').should('be.gt', 1).then(linesNum => {
-            cy.get(`${Cypress.env('bodyJsonClass')} > >`).its('length').should('be.gt', linesNum).then(jsonItemsNum => {
-                checkPrettyAndLineNums(jsonItemsNum, decodedBody);
-
-                clickCheckbox('Line numbers');
-                checkPrettyOrNothing(jsonItemsNum, decodedBody);
-
-                clickCheckbox('Pretty');
-                checkPrettyOrNothing(jsonItemsNum, decodedBody);
-
-                clickCheckbox('Line numbers');
-                checkOnlyLineNumberes(jsonItemsNum, decodedBody);
-            });
-        });
-    });
-}
-
-function clickCheckbox(type) {
-    cy.contains(`${type}`).prev().children().click();
-}
-
-function checkPrettyAndLineNums(jsonItemsLen, decodedBody) {
-    decodedBody = decodedBody.replaceAll(' ', '');
-    cy.get(`${Cypress.env('bodyJsonClass')} >`).then(elements => {
-        const lines = Object.values(elements);
-        lines.forEach((line, index) => {
-            if (line.getAttribute) {
-                const cleanLine = getCleanLine(line);
-                const currentLineFromDecodedText = decodedBody.substring(0, cleanLine.length);
-
-                expect(cleanLine).to.equal(currentLineFromDecodedText, `expected the text in line number ${index + 1} to match the text that generated by the base64 decoding`)
-
-                decodedBody = decodedBody.substring(cleanLine.length);
-            }
-        });
-    });
-}
-
-function getCleanLine(lineElement) {
-    return (lineElement.innerText.substring(0, lineElement.innerText.length - 1)).replaceAll(' ', '');
-}
-
-function checkPrettyOrNothing(jsonItems, decodedBody) {
-    cy.get(`${Cypress.env('bodyJsonClass')} > `).should('have.length', jsonItems).then(text => {
-        const json = text.text();
-        expect(json).to.equal(decodedBody);
-    });
-}
-
-function checkOnlyLineNumberes(jsonItems, decodedText) {
-    cy.get(`${Cypress.env('bodyJsonClass')} >`).should('have.length', 1).and('have.text', decodedText);
-    cy.get(`${Cypress.env('bodyJsonClass')} > >`).should('have.length', jsonItems)
-}
-
-function serviceMapCheck() {
-    it('service map test', function () {
-        cy.intercept(`${Cypress.env('testUrl')}/servicemap/get`).as('serviceMapRequest');
-        cy.get('#total-entries').should('not.have.text', '0').then(() => {
-            cy.get('#total-entries').invoke('text').then(entriesNum => {
-                cy.get('[alt="service-map"]').click();
-                cy.wait('@serviceMapRequest').then(({response}) => {
-                    const body = response.body;
-                    const nodeParams = {
-                        destination: 'httpbin.mizu-tests',
-                        source: '127.0.0.1'
-                    };
-                    serviceMapAPICheck(body, parseInt(entriesNum), nodeParams);
-                    cy.reload();
-                });
-            });
-        });
-    });
-}
-
-function serviceMapAPICheck(body, entriesNum, nodeParams) {
-    const {nodes, edges} = body;
-
-    expect(nodes.length).to.equal(Object.keys(nodeParams).length, `Expected nodes count`);
-
-    expect(edges.some(edge => edge.source.name === nodeParams.source)).to.be.true;
-    expect(edges.some(edge => edge.destination.name === nodeParams.destination)).to.be.true;
-
-    let count = 0;
-    edges.forEach(edge => {
-        count += edge.count;
-        if (edge.destination.name === nodeParams.destination) {
-            expect(edge.source.name).to.equal(nodeParams.source);
-        }
-    });
-
-    expect(count).to.equal(entriesNum);
-}

acceptanceTests/extensions_test.go

@@ -1,240 +0,0 @@
-package acceptanceTests
-
-import (
-	"context"
-	"fmt"
-	"github.com/go-redis/redis/v8"
-	amqp "github.com/rabbitmq/amqp091-go"
-	"os/exec"
-	"testing"
-	"time"
-)
-
-func TestRedis(t *testing.T) {
-	if testing.Short() {
-		t.Skip("ignored acceptance test")
-	}
-
-	cliPath, cliPathErr := GetCliPath()
-	if cliPathErr != nil {
-		t.Errorf("failed to get cli path, err: %v", cliPathErr)
-		return
-	}
-
-	tapCmdArgs := GetDefaultTapCommandArgs()
-
-	tapNamespace := GetDefaultTapNamespace()
-	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-
-	tapCmd := exec.Command(cliPath, tapCmdArgs...)
-	t.Logf("running command: %v", tapCmd.String())
-
-	t.Cleanup(func() {
-		if err := CleanupCommand(tapCmd); err != nil {
-			t.Logf("failed to cleanup tap command, err: %v", err)
-		}
-	})
-
-	if err := tapCmd.Start(); err != nil {
-		t.Errorf("failed to start tap command, err: %v", err)
-		return
-	}
-
-	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
-
-	if err := WaitTapPodsReady(apiServerUrl); err != nil {
-		t.Errorf("failed to start tap pods on time, err: %v", err)
-		return
-	}
-
-	ctx := context.Background()
-
-	kubernetesProvider, err := NewKubernetesProvider()
-	if err != nil {
-		t.Errorf("failed to create k8s provider, err %v", err)
-		return
-	}
-
-	redisExternalIp, err := kubernetesProvider.GetServiceExternalIp(ctx, DefaultNamespaceName, "redis")
-	if err != nil {
-		t.Errorf("failed to get redis external ip, err: %v", err)
-		return
-	}
-
-	rdb := redis.NewClient(&redis.Options{
-		Addr: fmt.Sprintf("%v:6379", redisExternalIp),
-	})
-
-	for i := 0; i < DefaultEntriesCount/5; i++ {
-		requestErr := rdb.Ping(ctx).Err()
-		if requestErr != nil {
-			t.Errorf("failed to send redis request, err: %v", requestErr)
-			return
-		}
-	}
-
-	for i := 0; i < DefaultEntriesCount/5; i++ {
-		requestErr := rdb.Set(ctx, "key", "value", -1).Err()
-		if requestErr != nil {
-			t.Errorf("failed to send redis request, err: %v", requestErr)
-			return
-		}
-	}
-
-	for i := 0; i < DefaultEntriesCount/5; i++ {
-		requestErr := rdb.Exists(ctx, "key").Err()
-		if requestErr != nil {
-			t.Errorf("failed to send redis request, err: %v", requestErr)
-			return
-		}
-	}
-
-	for i := 0; i < DefaultEntriesCount/5; i++ {
-		requestErr := rdb.Get(ctx, "key").Err()
-		if requestErr != nil {
-			t.Errorf("failed to send redis request, err: %v", requestErr)
-			return
-		}
-	}
-
-	for i := 0; i < DefaultEntriesCount/5; i++ {
-		requestErr := rdb.Del(ctx, "key").Err()
-		if requestErr != nil {
-			t.Errorf("failed to send redis request, err: %v", requestErr)
-			return
-		}
-	}
-
-	RunCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/Redis.js\"")
-}
-
-func TestAmqp(t *testing.T) {
-	if testing.Short() {
-		t.Skip("ignored acceptance test")
-	}
-
-	cliPath, cliPathErr := GetCliPath()
-	if cliPathErr != nil {
-		t.Errorf("failed to get cli path, err: %v", cliPathErr)
-		return
-	}
-
-	tapCmdArgs := GetDefaultTapCommandArgs()
-
-	tapNamespace := GetDefaultTapNamespace()
-	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-
-	tapCmd := exec.Command(cliPath, tapCmdArgs...)
-	t.Logf("running command: %v", tapCmd.String())
-
-	t.Cleanup(func() {
-		if err := CleanupCommand(tapCmd); err != nil {
-			t.Logf("failed to cleanup tap command, err: %v", err)
-		}
-	})
-
-	if err := tapCmd.Start(); err != nil {
-		t.Errorf("failed to start tap command, err: %v", err)
-		return
-	}
-
-	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
-
-	if err := WaitTapPodsReady(apiServerUrl); err != nil {
-		t.Errorf("failed to start tap pods on time, err: %v", err)
-		return
-	}
-
-	ctx := context.Background()
-
-	kubernetesProvider, err := NewKubernetesProvider()
-	if err != nil {
-		t.Errorf("failed to create k8s provider, err %v", err)
-		return
-	}
-
-	rabbitmqExternalIp, err := kubernetesProvider.GetServiceExternalIp(ctx, DefaultNamespaceName, "rabbitmq")
-	if err != nil {
-		t.Errorf("failed to get RabbitMQ external ip, err: %v", err)
-		return
-	}
-
-	conn, err := amqp.Dial(fmt.Sprintf("amqp://guest:guest@%v:5672/", rabbitmqExternalIp))
-	if err != nil {
-		t.Errorf("failed to connect to RabbitMQ, err: %v", err)
-		return
-	}
-	defer conn.Close()
-
-	// Temporary fix for missing amqp entries
-	time.Sleep(10 * time.Second)
-
-	for i := 0; i < DefaultEntriesCount/5; i++ {
-		ch, err := conn.Channel()
-		if err != nil {
-			t.Errorf("failed to open a channel, err: %v", err)
-			return
-		}
-
-		exchangeName := "exchange"
-		err = ch.ExchangeDeclare(exchangeName, "direct", true, false, false, false, nil)
-		if err != nil {
-			t.Errorf("failed to declare an exchange, err: %v", err)
-			return
-		}
-
-		q, err := ch.QueueDeclare("queue", true, false, false, false, nil)
-		if err != nil {
-			t.Errorf("failed to declare a queue, err: %v", err)
-			return
-		}
-
-		routingKey := "routing_key"
-		err = ch.QueueBind(q.Name, routingKey, exchangeName, false, nil)
-		if err != nil {
-			t.Errorf("failed to bind the queue, err: %v", err)
-			return
-		}
-
-		err = ch.Publish(exchangeName, routingKey, false, false,
-			amqp.Publishing{
-				DeliveryMode: amqp.Persistent,
-				ContentType:  "text/plain",
-				Body:         []byte("message"),
-			})
-		if err != nil {
-			t.Errorf("failed to publish a message, err: %v", err)
-			return
-		}
-
-		msgChan, err := ch.Consume(q.Name, "Consumer", true, false, false, false, nil)
-		if err != nil {
-			t.Errorf("failed to create a consumer, err: %v", err)
-			return
-		}
-
-		select {
-		case <-msgChan:
-			break
-		case <-time.After(3 * time.Second):
-			t.Errorf("failed to consume a message on time")
-			return
-		}
-
-		err = ch.ExchangeDelete(exchangeName, false, false)
-		if err != nil {
-			t.Errorf("failed to delete the exchange, err: %v", err)
-			return
-		}
-
-		_, err = ch.QueueDelete(q.Name, false, false, false)
-		if err != nil {
-			t.Errorf("failed to delete the queue, err: %v", err)
-			return
-		}
-
-		ch.Close()
-	}
-
-	RunCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/Rabbit.js\"")
-}

acceptanceTests/go.mod

@@ -1,51 +1,11 @@
-module github.com/up9inc/mizu/acceptanceTests
+module github.com/up9inc/mizu/tests
 
-go 1.17
+go 1.16
 
 require (
-	github.com/go-redis/redis/v8 v8.11.4
-	github.com/rabbitmq/amqp091-go v1.3.0
+	github.com/gorilla/websocket v1.4.2
 	github.com/up9inc/mizu/shared v0.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
-	k8s.io/apimachinery v0.23.3
-	k8s.io/client-go v0.23.3
-)
-
-require (
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
-	github.com/go-logr/logr v1.2.2 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.7 // indirect
-	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/api v0.23.3 // indirect
-	k8s.io/klog/v2 v2.40.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf // indirect
-	k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect
-	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared

acceptanceTests/logs_test.go

@@ -11,22 +11,22 @@ func TestLogs(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := GetCliPath()
+	cliPath, cliPathErr := getCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := GetDefaultTapCommandArgs()
+	tapCmdArgs := getDefaultTapCommandArgs()
 
-	tapNamespace := GetDefaultTapNamespace()
+	tapNamespace := getDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := CleanupCommand(tapCmd); err != nil {
+		if err := cleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -36,14 +36,14 @@ func TestLogs(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
 
-	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	logsCmdArgs := GetDefaultLogsCommandArgs()
+	logsCmdArgs := getDefaultLogsCommandArgs()
 
 	logsCmd := exec.Command(cliPath, logsCmdArgs...)
 	t.Logf("running command: %v", logsCmd.String())
@@ -58,7 +58,7 @@ func TestLogs(t *testing.T) {
 		return
 	}
 
-	logsPath, logsPathErr := GetLogsPath()
+	logsPath, logsPathErr := getLogsPath()
 	if logsPathErr != nil {
 		t.Errorf("failed to get logs path, err: %v", logsPathErr)
 		return
@@ -112,22 +112,22 @@ func TestLogsPath(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := GetCliPath()
+	cliPath, cliPathErr := getCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := GetDefaultTapCommandArgs()
+	tapCmdArgs := getDefaultTapCommandArgs()
 
-	tapNamespace := GetDefaultTapNamespace()
+	tapNamespace := getDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := CleanupCommand(tapCmd); err != nil {
+		if err := cleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -137,14 +137,14 @@ func TestLogsPath(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
 
-	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	logsCmdArgs := GetDefaultLogsCommandArgs()
+	logsCmdArgs := getDefaultLogsCommandArgs()
 
 	logsPath := "../logs.zip"
 	logsCmdArgs = append(logsCmdArgs, "-f", logsPath)

acceptanceTests/setup.sh

@@ -36,24 +36,12 @@ kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests
 
 kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2
 
-echo "Creating redis deployment"
-kubectl create deployment redis --image=redis -n mizu-tests
-
-echo "Creating rabbitmq deployment"
-kubectl create deployment rabbitmq --image=rabbitmq -n mizu-tests
-
 echo "Creating httpbin services"
 kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests
 kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests
 
 kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2
 
-echo "Creating redis service"
-kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests
-
-echo "Creating rabbitmq service"
-kubectl expose deployment rabbitmq --type=LoadBalancer --port=5672 -n mizu-tests
-
 echo "Starting proxy"
 kubectl proxy --port=8080 &
 
@@ -61,10 +49,7 @@ echo "Setting minikube docker env"
 eval $(minikube docker-env)
 
 echo "Build agent image"
-docker build -t mizu/ci:0.0 .
+make build-docker-ci
 
 echo "Build cli"
-cd cli && make build GIT_BRANCH=ci SUFFIX=ci
-
-echo "Starting tunnel"
-minikube tunnel &
+make build-cli-ci

acceptanceTests/tap_test.go

@@ -14,10 +14,6 @@ import (
 )
 
 func TestTap(t *testing.T) {
-	basicTapTest(t, false)
-}
-
-func basicTapTest(t *testing.T, shouldCheckSrcAndDest bool, extraArgs... string) {
 	if testing.Short() {
 		t.Skip("ignored acceptance test")
 	}
@@ -26,24 +22,22 @@ func basicTapTest(t *testing.T, shouldCheckSrcAndDest bool, extraArgs... string)
 
 	for _, entriesCount := range tests {
 		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
-			cliPath, cliPathErr := GetCliPath()
+			cliPath, cliPathErr := getCliPath()
 			if cliPathErr != nil {
 				t.Errorf("failed to get cli path, err: %v", cliPathErr)
 				return
 			}
 
-			tapCmdArgs := GetDefaultTapCommandArgs()
+			tapCmdArgs := getDefaultTapCommandArgs()
 
-			tapNamespace := GetDefaultTapNamespace()
+			tapNamespace := getDefaultTapNamespace()
 			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
-			tapCmdArgs = append(tapCmdArgs, extraArgs...)
-
 			tapCmd := exec.Command(cliPath, tapCmdArgs...)
 			t.Logf("running command: %v", tapCmd.String())
 
 			t.Cleanup(func() {
-				if err := CleanupCommand(tapCmd); err != nil {
+				if err := cleanupCommand(tapCmd); err != nil {
 					t.Logf("failed to cleanup tap command, err: %v", err)
 				}
 			})
@@ -53,33 +47,50 @@ func basicTapTest(t *testing.T, shouldCheckSrcAndDest bool, extraArgs... string)
 				return
 			}
 
-			apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+			apiServerUrl := getApiServerUrl(defaultApiServerPort)
 
-			if err := WaitTapPodsReady(apiServerUrl); err != nil {
+			if err := waitTapPodsReady(apiServerUrl); err != nil {
 				t.Errorf("failed to start tap pods on time, err: %v", err)
 				return
 			}
 
-			proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
+			proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
 			for i := 0; i < entriesCount; i++ {
-				if _, requestErr := ExecuteHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
+				if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
 					t.Errorf("failed to send proxy request, err: %v", requestErr)
 					return
 				}
 			}
 
-			expectedPods := []PodDescriptor{
-				{Name: "httpbin", Namespace: "mizu-tests"},
-				{Name: "httpbin2", Namespace: "mizu-tests"},
-			}
+			entriesCheckFunc := func() error {
+				timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-			var expectedPodsStr string
-			for i := 0; i < len(expectedPods); i++ {
-				expectedPodsStr += fmt.Sprintf("Name:%vNamespace:%v", expectedPods[i].Name, expectedPods[i].Namespace)
-			}
+				entries, err := getDBEntries(timestamp, entriesCount, 1*time.Second)
+				if err != nil {
+					return err
+				}
+				err = checkEntriesAtLeast(entries, 1)
+				if err != nil {
+					return err
+				}
+				entry := entries[0]
 
-			RunCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/UiTest.js\" --env entriesCount=%d,arrayDict=%v,shouldCheckSrcAndDest=%v",
-				entriesCount, expectedPodsStr, shouldCheckSrcAndDest))
+				entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entry["id"])
+				requestResult, requestErr := executeHttpGetRequest(entryUrl)
+				if requestErr != nil {
+					return fmt.Errorf("failed to get entry, err: %v", requestErr)
+				}
+
+				if requestResult == nil {
+					return fmt.Errorf("unexpected nil entry result")
+				}
+
+				return nil
+			}
+			if err := retriesExecute(shortRetriesCount, entriesCheckFunc); err != nil {
+				t.Errorf("%v", err)
+				return
+			}
 		})
 	}
 }
@@ -93,15 +104,15 @@ func TestTapGuiPort(t *testing.T) {
 
 	for _, guiPort := range tests {
 		t.Run(fmt.Sprintf("%d", guiPort), func(t *testing.T) {
-			cliPath, cliPathErr := GetCliPath()
+			cliPath, cliPathErr := getCliPath()
 			if cliPathErr != nil {
 				t.Errorf("failed to get cli path, err: %v", cliPathErr)
 				return
 			}
 
-			tapCmdArgs := GetDefaultTapCommandArgs()
+			tapCmdArgs := getDefaultTapCommandArgs()
 
-			tapNamespace := GetDefaultTapNamespace()
+			tapNamespace := getDefaultTapNamespace()
 			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 			tapCmdArgs = append(tapCmdArgs, "-p", fmt.Sprintf("%d", guiPort))
@@ -110,7 +121,7 @@ func TestTapGuiPort(t *testing.T) {
 			t.Logf("running command: %v", tapCmd.String())
 
 			t.Cleanup(func() {
-				if err := CleanupCommand(tapCmd); err != nil {
+				if err := cleanupCommand(tapCmd); err != nil {
 					t.Logf("failed to cleanup tap command, err: %v", err)
 				}
 			})
@@ -120,23 +131,14 @@ func TestTapGuiPort(t *testing.T) {
 				return
 			}
 
-			apiServerUrl := GetApiServerUrl(guiPort)
+			apiServerUrl := getApiServerUrl(guiPort)
 
-			if err := WaitTapPodsReady(apiServerUrl); err != nil {
+			if err := waitTapPodsReady(apiServerUrl); err != nil {
 				t.Errorf("failed to start tap pods on time, err: %v", err)
 				return
 			}
 
-			proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
-			for i := 0; i < DefaultEntriesCount; i++ {
-				if _, requestErr := ExecuteHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
-					t.Errorf("failed to send proxy request, err: %v", requestErr)
-					return
-				}
-			}
-
-			RunCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/GuiPort.js\" --env name=%v,namespace=%v,port=%d",
-				"httpbin", "mizu-tests", guiPort))
+			runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/GuiPort.js\" --env port=%d --config-file cypress/integration/configurations/Default.json", guiPort))
 		})
 	}
 }
@@ -152,20 +154,20 @@ func TestTapAllNamespaces(t *testing.T) {
 		{Name: "httpbin", Namespace: "mizu-tests2"},
 	}
 
-	cliPath, cliPathErr := GetCliPath()
+	cliPath, cliPathErr := getCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := GetDefaultTapCommandArgs()
+	tapCmdArgs := getDefaultTapCommandArgs()
 	tapCmdArgs = append(tapCmdArgs, "-A")
 
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := CleanupCommand(tapCmd); err != nil {
+		if err := cleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -175,14 +177,14 @@ func TestTapAllNamespaces(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
 
-	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	RunCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v --config-file cypress/integration/configurations/Default.json",
 		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
 }
 
@@ -197,13 +199,13 @@ func TestTapMultipleNamespaces(t *testing.T) {
 		{Name: "httpbin", Namespace: "mizu-tests2"},
 	}
 
-	cliPath, cliPathErr := GetCliPath()
+	cliPath, cliPathErr := getCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := GetDefaultTapCommandArgs()
+	tapCmdArgs := getDefaultTapCommandArgs()
 	var namespacesCmd []string
 	for _, expectedPod := range expectedPods {
 		namespacesCmd = append(namespacesCmd, "-n", expectedPod.Namespace)
@@ -214,7 +216,7 @@ func TestTapMultipleNamespaces(t *testing.T) {
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := CleanupCommand(tapCmd); err != nil {
+		if err := cleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -224,14 +226,14 @@ func TestTapMultipleNamespaces(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
 
-	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	RunCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v --config-file cypress/integration/configurations/Default.json",
 		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
 }
 
@@ -245,22 +247,22 @@ func TestTapRegex(t *testing.T) {
 		{Name: regexPodName, Namespace: "mizu-tests"},
 	}
 
-	cliPath, cliPathErr := GetCliPath()
+	cliPath, cliPathErr := getCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := GetDefaultTapCommandArgsWithRegex(regexPodName)
+	tapCmdArgs := getDefaultTapCommandArgsWithRegex(regexPodName)
 
-	tapNamespace := GetDefaultTapNamespace()
+	tapNamespace := getDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := CleanupCommand(tapCmd); err != nil {
+		if err := cleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -270,14 +272,14 @@ func TestTapRegex(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
 
-	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	RunCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/Regex.js\" --env name=%v,namespace=%v",
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/Regex.js\" --env name=%v,namespace=%v --config-file cypress/integration/configurations/Default.json",
 		expectedPods[0].Name, expectedPods[0].Namespace))
 }
 
@@ -286,15 +288,15 @@ func TestTapDryRun(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := GetCliPath()
+	cliPath, cliPathErr := getCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := GetDefaultTapCommandArgs()
+	tapCmdArgs := getDefaultTapCommandArgs()
 
-	tapNamespace := GetDefaultTapNamespace()
+	tapNamespace := getDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 	tapCmdArgs = append(tapCmdArgs, "--dry-run")
@@ -318,7 +320,7 @@ func TestTapDryRun(t *testing.T) {
 	}()
 
 	go func() {
-		time.Sleep(ShortRetriesCount * time.Second)
+		time.Sleep(shortRetriesCount * time.Second)
 		resultChannel <- "fail"
 	}()
 
@@ -333,22 +335,22 @@ func TestTapRedact(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := GetCliPath()
+	cliPath, cliPathErr := getCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := GetDefaultTapCommandArgs()
+	tapCmdArgs := getDefaultTapCommandArgs()
 
-	tapNamespace := GetDefaultTapNamespace()
+	tapNamespace := getDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := CleanupCommand(tapCmd); err != nil {
+		if err := cleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -358,24 +360,24 @@ func TestTapRedact(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
 
-	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
+	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
 	requestHeaders := map[string]string{"User-Header": "Mizu"}
 	requestBody := map[string]string{"User": "Mizu"}
-	for i := 0; i < DefaultEntriesCount; i++ {
-		if _, requestErr := ExecuteHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
+	for i := 0; i < defaultEntriesCount; i++ {
+		if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
 	}
 
-	RunCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/Redact.js\"")
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/Redact.js\" --config-file cypress/integration/configurations/Default.json"))
 }
 
 func TestTapNoRedact(t *testing.T) {
@@ -383,15 +385,15 @@ func TestTapNoRedact(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := GetCliPath()
+	cliPath, cliPathErr := getCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := GetDefaultTapCommandArgs()
+	tapCmdArgs := getDefaultTapCommandArgs()
 
-	tapNamespace := GetDefaultTapNamespace()
+	tapNamespace := getDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 	tapCmdArgs = append(tapCmdArgs, "--no-redact")
@@ -400,7 +402,7 @@ func TestTapNoRedact(t *testing.T) {
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := CleanupCommand(tapCmd); err != nil {
+		if err := cleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -410,24 +412,24 @@ func TestTapNoRedact(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
 
-	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
+	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
 	requestHeaders := map[string]string{"User-Header": "Mizu"}
 	requestBody := map[string]string{"User": "Mizu"}
-	for i := 0; i < DefaultEntriesCount; i++ {
-		if _, requestErr := ExecuteHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
+	for i := 0; i < defaultEntriesCount; i++ {
+		if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
 	}
 
-	RunCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/NoRedact.js\"")
+	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/NoRedact.js\" --config-file cypress/integration/configurations/Default.json")
 }
 
 func TestTapRegexMasking(t *testing.T) {
@@ -435,15 +437,15 @@ func TestTapRegexMasking(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := GetCliPath()
+	cliPath, cliPathErr := getCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := GetDefaultTapCommandArgs()
+	tapCmdArgs := getDefaultTapCommandArgs()
 
-	tapNamespace := GetDefaultTapNamespace()
+	tapNamespace := getDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 	tapCmdArgs = append(tapCmdArgs, "-r", "Mizu")
@@ -452,7 +454,7 @@ func TestTapRegexMasking(t *testing.T) {
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := CleanupCommand(tapCmd); err != nil {
+		if err := cleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -462,23 +464,23 @@ func TestTapRegexMasking(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
 
-	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
-	for i := 0; i < DefaultEntriesCount; i++ {
+	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+	for i := 0; i < defaultEntriesCount; i++ {
 		response, requestErr := http.Post(fmt.Sprintf("%v/post", proxyUrl), "text/plain", bytes.NewBufferString("Mizu"))
-		if _, requestErr = ExecuteHttpRequest(response, requestErr); requestErr != nil {
+		if _, requestErr = executeHttpRequest(response, requestErr); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
 	}
 
-	RunCypressTests(t, "npx cypress run --spec \"cypress/integration/tests/RegexMasking.js\"")
+	runCypressTests(t, "npx cypress run --spec \"cypress/integration/tests/RegexMasking.js\" --config-file cypress/integration/configurations/Default.json")
 
 }
 
@@ -487,15 +489,15 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := GetCliPath()
+	cliPath, cliPathErr := getCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := GetDefaultTapCommandArgs()
+	tapCmdArgs := getDefaultTapCommandArgs()
 
-	tapNamespace := GetDefaultTapNamespace()
+	tapNamespace := getDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 	ignoredUserAgentValue := "ignore"
@@ -505,7 +507,7 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := CleanupCommand(tapCmd); err != nil {
+		if err := cleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -515,32 +517,32 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
 
-	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
+	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
 
 	ignoredUserAgentCustomHeader := "Ignored-User-Agent"
 	headers := map[string]string{"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""}
-	for i := 0; i < DefaultEntriesCount; i++ {
-		if _, requestErr := ExecuteHttpGetRequestWithHeaders(fmt.Sprintf("%v/get", proxyUrl), headers); requestErr != nil {
+	for i := 0; i < defaultEntriesCount; i++ {
+		if _, requestErr := executeHttpGetRequestWithHeaders(fmt.Sprintf("%v/get", proxyUrl), headers); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
 	}
 
-	for i := 0; i < DefaultEntriesCount; i++ {
-		if _, requestErr := ExecuteHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
+	for i := 0; i < defaultEntriesCount; i++ {
+		if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
 	}
 
-	RunCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/IgnoredUserAgents.js\"")
+	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/IgnoredUserAgents.js\" --config-file cypress/integration/configurations/HugeMizu.json")
 }
 
 func TestTapDumpLogs(t *testing.T) {
@@ -548,15 +550,15 @@ func TestTapDumpLogs(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := GetCliPath()
+	cliPath, cliPathErr := getCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := GetDefaultTapCommandArgs()
+	tapCmdArgs := getDefaultTapCommandArgs()
 
-	tapNamespace := GetDefaultTapNamespace()
+	tapNamespace := getDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 	tapCmdArgs = append(tapCmdArgs, "--set", "dump-logs=true")
@@ -569,19 +571,19 @@ func TestTapDumpLogs(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
 
-	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	if err := CleanupCommand(tapCmd); err != nil {
+	if err := cleanupCommand(tapCmd); err != nil {
 		t.Errorf("failed to cleanup tap command, err: %v", err)
 		return
 	}
 
-	mizuFolderPath, mizuPathErr := GetMizuFolderPath()
+	mizuFolderPath, mizuPathErr := getMizuFolderPath()
 	if mizuPathErr != nil {
 		t.Errorf("failed to get mizu folder path, err: %v", mizuPathErr)
 		return
@@ -649,44 +651,3 @@ func TestTapDumpLogs(t *testing.T) {
 		return
 	}
 }
-
-func TestIpResolving(t *testing.T) {
-	namespace := AllNamespaces
-
-	t.Log("add permissions for ip-resolution for current user")
-	if err := ApplyKubeFilesForTest(
-		t,
-		"minikube",
-		namespace,
-		"../cli/cmd/permissionFiles/permissions-all-namespaces-ip-resolution-optional.yaml",
-	); err != nil {
-		t.Errorf("failed to create k8s permissions, %v", err)
-		return
-	}
-
-	basicTapTest(t, true)
-}
-
-func TestRestrictedMode(t *testing.T) {
-	namespace := "mizu-tests"
-
-	t.Log("creating permissions for restricted user")
-	if err := ApplyKubeFilesForTest(
-		t,
-		"minikube",
-		namespace,
-		"../cli/cmd/permissionFiles/permissions-ns-tap.yaml",
-	); err != nil {
-		t.Errorf("failed to create k8s permissions, %v", err)
-		return
-	}
-
-	t.Log("switching k8s context to user")
-	if err := SwitchKubeContextForTest(t, "user-with-restricted-access"); err != nil {
-		t.Errorf("failed to switch k8s context, %v", err)
-		return
-	}
-
-	extraArgs := []string{"--set", fmt.Sprintf("mizu-resources-namespace=%s", namespace)}
-	t.Run("basic tap", func (testingT *testing.T) {basicTapTest(testingT, false, extraArgs...)})
-}

acceptanceTests/testsUtils.go

@@ -2,36 +2,33 @@ package acceptanceTests
 
 import (
 	"bytes"
-	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/util/homedir"
 	"net/http"
 	"os"
 	"os/exec"
 	"path"
-	"path/filepath"
 	"strings"
+	"sync"
 	"syscall"
 	"testing"
 	"time"
 
+	"github.com/gorilla/websocket"
 	"github.com/up9inc/mizu/shared"
 )
 
 const (
-	LongRetriesCount      = 100
-	ShortRetriesCount     = 10
-	DefaultApiServerPort  = shared.DefaultApiServerPort
-	DefaultNamespaceName  = "mizu-tests"
-	DefaultServiceName    = "httpbin"
-	DefaultEntriesCount   = 50
-	WaitAfterTapPodsReady = 3 * time.Second
-	AllNamespaces         = ""
+	longRetriesCount      = 100
+	shortRetriesCount     = 10
+	defaultApiServerPort  = shared.DefaultApiServerPort
+	defaultNamespaceName  = "mizu-tests"
+	defaultServiceName    = "httpbin"
+	defaultEntriesCount   = 50
+	waitAfterTapPodsReady = 3 * time.Second
+	cleanCommandTimeout   = 1 * time.Minute
 )
 
 type PodDescriptor struct {
@@ -39,7 +36,19 @@ type PodDescriptor struct {
 	Namespace string
 }
 
-func GetCliPath() (string, error) {
+func isPodDescriptorInPodArray(pods []map[string]interface{}, podDescriptor PodDescriptor) bool {
+	for _, pod := range pods {
+		podNamespace := pod["namespace"].(string)
+		podName := pod["name"].(string)
+
+		if podDescriptor.Namespace == podNamespace && strings.Contains(podName, podDescriptor.Name) {
+			return true
+		}
+	}
+	return false
+}
+
+func getCliPath() (string, error) {
 	dir, filePathErr := os.Getwd()
 	if filePathErr != nil {
 		return "", filePathErr
@@ -49,7 +58,7 @@ func GetCliPath() (string, error) {
 	return cliPath, nil
 }
 
-func GetMizuFolderPath() (string, error) {
+func getMizuFolderPath() (string, error) {
 	home, homeDirErr := os.UserHomeDir()
 	if homeDirErr != nil {
 		return "", homeDirErr
@@ -58,8 +67,8 @@ func GetMizuFolderPath() (string, error) {
 	return path.Join(home, ".mizu"), nil
 }
 
-func GetConfigPath() (string, error) {
-	mizuFolderPath, mizuPathError := GetMizuFolderPath()
+func getConfigPath() (string, error) {
+	mizuFolderPath, mizuPathError := getMizuFolderPath()
 	if mizuPathError != nil {
 		return "", mizuPathError
 	}
@@ -67,209 +76,90 @@ func GetConfigPath() (string, error) {
 	return path.Join(mizuFolderPath, "config.yaml"), nil
 }
 
-func GetProxyUrl(namespace string, service string) string {
+func getProxyUrl(namespace string, service string) string {
 	return fmt.Sprintf("http://localhost:8080/api/v1/namespaces/%v/services/%v/proxy", namespace, service)
 }
 
-func GetApiServerUrl(port uint16) string {
+func getApiServerUrl(port uint16) string {
 	return fmt.Sprintf("http://localhost:%v", port)
 }
 
-func NewKubernetesProvider() (*KubernetesProvider, error) {
-	home := homedir.HomeDir()
-	configLoadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(home, ".kube", "config")}
-	clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
-		configLoadingRules,
-		&clientcmd.ConfigOverrides{
-			CurrentContext: "",
-		},
-	)
-
-	restClientConfig, err := clientConfig.ClientConfig()
-	if err != nil {
-		return nil, err
-	}
-
-	clientSet, err := kubernetes.NewForConfig(restClientConfig)
-	if err != nil {
-		return nil, err
-	}
-
-	return &KubernetesProvider{clientSet}, nil
-}
-
-type KubernetesProvider struct {
-	clientSet *kubernetes.Clientset
-}
-
-func (kp *KubernetesProvider) GetServiceExternalIp(ctx context.Context, namespace string, service string) (string, error) {
-	serviceObj, err := kp.clientSet.CoreV1().Services(namespace).Get(ctx, service, metav1.GetOptions{})
-	if err != nil {
-		return "", err
-	}
-
-	externalIp := serviceObj.Status.LoadBalancer.Ingress[0].IP
-	return externalIp, nil
-}
-
-func SwitchKubeContextForTest(t *testing.T, newContextName string) error {
-	prevKubeContextName, err := GetKubeCurrentContextName()
-	if err != nil {
-		return err
-	}
-
-	if err := SetKubeCurrentContext(newContextName); err != nil {
-		return err
-	}
-
-	t.Cleanup(func() {
-		if err := SetKubeCurrentContext(prevKubeContextName); err != nil {
-			t.Errorf("failed to set Kubernetes context to %s, err: %v", prevKubeContextName, err)
-			t.Errorf("cleanup failed, subsequent tests may be affected")
-		}
-	})
-
-	return nil
-}
-
-func GetKubeCurrentContextName() (string, error) {
-	cmd := exec.Command("kubectl", "config", "current-context")
-
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		return "", fmt.Errorf("%v, %s", err, string(output))
-	}
-
-	return string(bytes.TrimSpace(output)), nil
-}
-
-func SetKubeCurrentContext(contextName string) error {
-	cmd := exec.Command("kubectl", "config", "use-context", contextName)
-
-	if output, err := cmd.CombinedOutput(); err != nil {
-		return fmt.Errorf("%v, %s", err, string(output))
-	}
-
-	return nil
-}
-
-func ApplyKubeFilesForTest(t *testing.T, kubeContext string, namespace string, filename ...string) error {
-	for i := range filename {
-		fname := filename[i]
-		if err := ApplyKubeFile(kubeContext, namespace, fname); err != nil {
-			return err
-		}
-
-		t.Cleanup(func() {
-			if err := DeleteKubeFile(kubeContext, namespace, fname); err != nil {
-				t.Errorf(
-					"failed to delete Kubernetes resources in namespace %s from filename %s, err: %v",
-					namespace,
-					fname,
-					err,
-				)
-			}
-		})
-	}
-
-	return nil
-}
-
-func ApplyKubeFile(kubeContext string, namespace string, filename string) (error) {
-	cmdArgs := []string{
-		"apply",
-		"--context", kubeContext,
-		"-f", filename,
-	}
-	if namespace != AllNamespaces {
-		cmdArgs = append(cmdArgs, "-n", namespace)
-	}
-	cmd := exec.Command("kubectl", cmdArgs...)
-
-	if output, err := cmd.CombinedOutput(); err != nil {
-		return fmt.Errorf("%v, %s", err, string(output))
-	}
-
-	return nil
-}
-
-func DeleteKubeFile(kubeContext string, namespace string, filename string) error {
-	cmdArgs := []string{
-		"delete",
-		"--context", kubeContext,
-		"-f", filename,
-	}
-	if namespace != AllNamespaces {
-		cmdArgs = append(cmdArgs, "-n", namespace)
-	}
-	cmd := exec.Command("kubectl", cmdArgs...)
-
-	if output, err := cmd.CombinedOutput(); err != nil {
-		return fmt.Errorf("%v, %s", err, string(output))
-	}
-
-	return nil
+func getWebSocketUrl(port uint16) string {
+	return fmt.Sprintf("ws://localhost:%v/ws", port)
 }
 
 func getDefaultCommandArgs() []string {
 	setFlag := "--set"
 	telemetry := "telemetry=false"
-	agentImage := "agent-image=mizu/ci:0.0"
+	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0"
 	imagePullPolicy := "image-pull-policy=IfNotPresent"
 	headless := "headless=true"
 
 	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
 }
 
-func GetDefaultTapCommandArgs() []string {
+func getDefaultTapCommandArgs() []string {
 	tapCommand := "tap"
 	defaultCmdArgs := getDefaultCommandArgs()
 
 	return append([]string{tapCommand}, defaultCmdArgs...)
 }
 
-func GetDefaultTapCommandArgsWithRegex(regex string) []string {
+func getDefaultTapCommandArgsWithRegex(regex string) []string {
 	tapCommand := "tap"
 	defaultCmdArgs := getDefaultCommandArgs()
 
 	return append([]string{tapCommand, regex}, defaultCmdArgs...)
 }
 
-func GetDefaultLogsCommandArgs() []string {
+func getDefaultLogsCommandArgs() []string {
 	logsCommand := "logs"
 	defaultCmdArgs := getDefaultCommandArgs()
 
 	return append([]string{logsCommand}, defaultCmdArgs...)
 }
 
-func GetDefaultTapNamespace() []string {
+func getDefaultTapNamespace() []string {
 	return []string{"-n", "mizu-tests"}
 }
 
-func GetDefaultConfigCommandArgs() []string {
+func getDefaultConfigCommandArgs() []string {
 	configCommand := "config"
 	defaultCmdArgs := getDefaultCommandArgs()
 
 	return append([]string{configCommand}, defaultCmdArgs...)
 }
 
-func RunCypressTests(t *testing.T, cypressRunCmd string) {
+func getDefaultCleanCommandArgs() []string {
+	cleanCommand := "clean"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{cleanCommand}, defaultCmdArgs...)
+}
+
+func getDefaultViewCommandArgs() []string {
+	viewCommand := "view"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{viewCommand}, defaultCmdArgs...)
+}
+
+func runCypressTests(t *testing.T, cypressRunCmd string) {
 	cypressCmd := exec.Command("bash", "-c", cypressRunCmd)
 	t.Logf("running command: %v", cypressCmd.String())
-	out, err := cypressCmd.CombinedOutput()
+	out, err := cypressCmd.Output()
 	if err != nil {
-		t.Errorf("error running cypress, error: %v, output: %v", err, string(out))
+		t.Errorf("%s", out)
 		return
 	}
-
 	t.Logf("%s", out)
 }
 
-func RetriesExecute(retriesCount int, executeFunc func() error) error {
+func retriesExecute(retriesCount int, executeFunc func() error) error {
 	var lastError interface{}
 
 	for i := 0; i < retriesCount; i++ {
-		if err := TryExecuteFunc(executeFunc); err != nil {
+		if err := tryExecuteFunc(executeFunc); err != nil {
 			lastError = err
 
 			time.Sleep(1 * time.Second)
@@ -282,7 +172,7 @@ func RetriesExecute(retriesCount int, executeFunc func() error) error {
 	return fmt.Errorf("reached max retries count, retries count: %v, last err: %v", retriesCount, lastError)
 }
 
-func TryExecuteFunc(executeFunc func() error) (err interface{}) {
+func tryExecuteFunc(executeFunc func() error) (err interface{}) {
 	defer func() {
 		if panicErr := recover(); panicErr != nil {
 			err = panicErr
@@ -292,10 +182,10 @@ func TryExecuteFunc(executeFunc func() error) (err interface{}) {
 	return executeFunc()
 }
 
-func WaitTapPodsReady(apiServerUrl string) error {
+func waitTapPodsReady(apiServerUrl string) error {
 	resolvingUrl := fmt.Sprintf("%v/status/connectedTappersCount", apiServerUrl)
 	tapPodsReadyFunc := func() error {
-		requestResult, requestErr := ExecuteHttpGetRequest(resolvingUrl)
+		requestResult, requestErr := executeHttpGetRequest(resolvingUrl)
 		if requestErr != nil {
 			return requestErr
 		}
@@ -304,14 +194,14 @@ func WaitTapPodsReady(apiServerUrl string) error {
 		if connectedTappersCount == 0 {
 			return fmt.Errorf("no connected tappers running")
 		}
-		time.Sleep(WaitAfterTapPodsReady)
+		time.Sleep(waitAfterTapPodsReady)
 		return nil
 	}
 
-	return RetriesExecute(LongRetriesCount, tapPodsReadyFunc)
+	return retriesExecute(longRetriesCount, tapPodsReadyFunc)
 }
 
-func JsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
+func jsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
 	var result interface{}
 	if parseErr := json.Unmarshal(jsonBytes, &result); parseErr != nil {
 		return nil, parseErr
@@ -320,7 +210,7 @@ func JsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
 	return result, nil
 }
 
-func ExecuteHttpRequest(response *http.Response, requestErr error) (interface{}, error) {
+func executeHttpRequest(response *http.Response, requestErr error) (interface{}, error) {
 	if requestErr != nil {
 		return nil, requestErr
 	} else if response.StatusCode != 200 {
@@ -334,10 +224,10 @@ func ExecuteHttpRequest(response *http.Response, requestErr error) (interface{},
 		return nil, readErr
 	}
 
-	return JsonBytesToInterface(data)
+	return jsonBytesToInterface(data)
 }
 
-func ExecuteHttpGetRequestWithHeaders(url string, headers map[string]string) (interface{}, error) {
+func executeHttpGetRequestWithHeaders(url string, headers map[string]string) (interface{}, error) {
 	request, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return nil, err
@@ -349,15 +239,15 @@ func ExecuteHttpGetRequestWithHeaders(url string, headers map[string]string) (in
 
 	client := &http.Client{}
 	response, requestErr := client.Do(request)
-	return ExecuteHttpRequest(response, requestErr)
+	return executeHttpRequest(response, requestErr)
 }
 
-func ExecuteHttpGetRequest(url string) (interface{}, error) {
+func executeHttpGetRequest(url string) (interface{}, error) {
 	response, requestErr := http.Get(url)
-	return ExecuteHttpRequest(response, requestErr)
+	return executeHttpRequest(response, requestErr)
 }
 
-func ExecuteHttpPostRequestWithHeaders(url string, headers map[string]string, body interface{}) (interface{}, error) {
+func executeHttpPostRequestWithHeaders(url string, headers map[string]string, body interface{}) (interface{}, error) {
 	requestBody, jsonErr := json.Marshal(body)
 	if jsonErr != nil {
 		return nil, jsonErr
@@ -375,10 +265,40 @@ func ExecuteHttpPostRequestWithHeaders(url string, headers map[string]string, bo
 
 	client := &http.Client{}
 	response, requestErr := client.Do(request)
-	return ExecuteHttpRequest(response, requestErr)
+	return executeHttpRequest(response, requestErr)
 }
 
-func CleanupCommand(cmd *exec.Cmd) error {
+func runMizuClean() error {
+	cliPath, err := getCliPath()
+	if err != nil {
+		return err
+	}
+
+	cleanCmdArgs := getDefaultCleanCommandArgs()
+
+	cleanCmd := exec.Command(cliPath, cleanCmdArgs...)
+
+	commandDone := make(chan error)
+	go func() {
+		if err := cleanCmd.Run(); err != nil {
+			commandDone <- err
+		}
+		commandDone <- nil
+	}()
+
+	select {
+	case err = <-commandDone:
+		if err != nil {
+			return err
+		}
+	case <-time.After(cleanCommandTimeout):
+		return errors.New("clean command timed out")
+	}
+
+	return nil
+}
+
+func cleanupCommand(cmd *exec.Cmd) error {
 	if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil {
 		return err
 	}
@@ -390,7 +310,18 @@ func CleanupCommand(cmd *exec.Cmd) error {
 	return nil
 }
 
-func GetLogsPath() (string, error) {
+func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) {
+	tapPodsInterface := tapStatusInterface.([]interface{})
+
+	var pods []map[string]interface{}
+	for _, podInterface := range tapPodsInterface {
+		pods = append(pods, podInterface.(map[string]interface{}))
+	}
+
+	return pods, nil
+}
+
+func getLogsPath() (string, error) {
 	dir, filePathErr := os.Getwd()
 	if filePathErr != nil {
 		return "", filePathErr
@@ -400,6 +331,77 @@ func GetLogsPath() (string, error) {
 	return logsPath, nil
 }
 
+// waitTimeout waits for the waitgroup for the specified max timeout.
+// Returns true if waiting timed out.
+func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
+	channel := make(chan struct{})
+	go func() {
+		defer close(channel)
+		wg.Wait()
+	}()
+	select {
+	case <-channel:
+		return false // completed normally
+	case <-time.After(timeout):
+		return true // timed out
+	}
+}
+
+// checkEntriesAtLeast checks whether the number of entries greater than or equal to n
+func checkEntriesAtLeast(entries []map[string]interface{}, n int) error {
+	if len(entries) < n {
+		return fmt.Errorf("Unexpected entries result - Expected more than %d entries", n-1)
+	}
+	return nil
+}
+
+// getDBEntries retrieves the entries from the database before the given timestamp.
+// Also limits the results according to the limit parameter.
+// Timeout for the WebSocket connection is defined by the timeout parameter.
+func getDBEntries(timestamp int64, limit int, timeout time.Duration) (entries []map[string]interface{}, err error) {
+	query := fmt.Sprintf("timestamp < %d and limit(%d)", timestamp, limit)
+	webSocketUrl := getWebSocketUrl(defaultApiServerPort)
+
+	var connection *websocket.Conn
+	connection, _, err = websocket.DefaultDialer.Dial(webSocketUrl, nil)
+	if err != nil {
+		return
+	}
+	defer connection.Close()
+
+	handleWSConnection := func(wg *sync.WaitGroup) {
+		defer wg.Done()
+		for {
+			_, message, err := connection.ReadMessage()
+			if err != nil {
+				return
+			}
+
+			var data map[string]interface{}
+			if err = json.Unmarshal([]byte(message), &data); err != nil {
+				return
+			}
+
+			if data["messageType"] == "entry" {
+				entries = append(entries, data)
+			}
+		}
+	}
+
+	err = connection.WriteMessage(websocket.TextMessage, []byte(query))
+	if err != nil {
+		return
+	}
+
+	var wg sync.WaitGroup
+	go handleWSConnection(&wg)
+	wg.Add(1)
+
+	waitTimeout(&wg, timeout)
+
+	return
+}
+
 func Contains(slice []string, containsValue string) bool {
 	for _, sliceValue := range slice {
 		if sliceValue == containsValue {

agent/go.mod

@@ -1,139 +1,34 @@
-module github.com/up9inc/mizu/agent
+module mizuserver
 
-go 1.17
+go 1.16
 
 require (
 	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
-	github.com/chanced/openapi v0.0.8
-	github.com/djherbis/atime v1.1.0
-	github.com/elastic/go-elasticsearch/v7 v7.17.0
-	github.com/getkin/kin-openapi v0.89.0
+	github.com/chanced/openapi v0.0.6
+	github.com/djherbis/atime v1.0.0
+	github.com/getkin/kin-openapi v0.76.0
 	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.7.7
-	github.com/go-playground/locales v0.14.0
-	github.com/go-playground/universal-translator v0.18.0
-	github.com/go-playground/validator/v10 v10.10.0
-	github.com/google/uuid v1.3.0
+	github.com/go-playground/locales v0.13.0
+	github.com/go-playground/universal-translator v0.17.0
+	github.com/go-playground/validator/v10 v10.5.0
+	github.com/google/martian v2.1.0+incompatible
+	github.com/google/uuid v1.1.2
 	github.com/gorilla/websocket v1.4.2
-	github.com/jinzhu/copier v0.3.5
-	github.com/nav-inc/datetime v0.1.3
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
-	github.com/orcaman/concurrent-map v1.0.0
+	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
+	github.com/ory/kratos-client-go v0.8.2-alpha.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/stretchr/testify v1.7.0
-	github.com/up9inc/basenine/client/go v0.0.0-20220413023528-c741e4aa1cf2
+	github.com/up9inc/basenine/client/go v0.0.0-20220117225014-f91ba3692a36
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
-	github.com/up9inc/mizu/tap/extensions/amqp v0.0.0
-	github.com/up9inc/mizu/tap/extensions/http v0.0.0
-	github.com/up9inc/mizu/tap/extensions/kafka v0.0.0
-	github.com/up9inc/mizu/tap/extensions/redis v0.0.0
-	github.com/wI2L/jsondiff v0.1.1
 	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
-	k8s.io/api v0.23.3
-	k8s.io/apimachinery v0.23.3
-	k8s.io/client-go v0.23.3
-)
-
-require (
-	cloud.google.com/go/compute v1.2.0 // indirect
-	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
-	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.24 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
-	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
-	github.com/Azure/go-autorest/logger v0.2.1 // indirect
-	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
-	github.com/MakeNowJust/heredoc v1.0.0 // indirect
-	github.com/PuerkitoBio/purell v1.1.1 // indirect
-	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
-	github.com/beevik/etree v1.1.0 // indirect
-	github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 // indirect
-	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
-	github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b // indirect
-	github.com/cilium/ebpf v0.8.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
-	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
-	github.com/fatih/camelcase v1.0.0 // indirect
-	github.com/fvbommel/sortorder v1.0.2 // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/gin-contrib/sse v0.1.0 // indirect
-	github.com/go-errors/errors v1.4.2 // indirect
-	github.com/go-logr/logr v1.2.2 // indirect
-	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.19.6 // indirect
-	github.com/go-openapi/swag v0.21.1 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/btree v1.0.1 // indirect
-	github.com/google/go-cmp v0.5.7 // indirect
-	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/gopacket v1.1.19 // indirect
-	github.com/google/martian v2.1.0+incompatible // indirect
-	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
-	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/josharian/intern v1.0.0 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.14.2 // indirect
-	github.com/leodido/go-urn v1.2.1 // indirect
-	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
-	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
-	github.com/moby/spdystream v0.2.0 // indirect
-	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
-	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
-	github.com/ohler55/ojg v1.12.12 // indirect
-	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
-	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/russross/blackfriday v1.6.0 // indirect
-	github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
-	github.com/segmentio/kafka-go v0.4.27 // indirect
-	github.com/spf13/cobra v1.3.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/tidwall/gjson v1.14.0 // indirect
-	github.com/tidwall/match v1.1.1 // indirect
-	github.com/tidwall/pretty v1.2.0 // indirect
-	github.com/tidwall/sjson v1.2.4 // indirect
-	github.com/ugorji/go/codec v1.2.6 // indirect
-	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
-	github.com/xlab/treeprint v1.1.0 // indirect
-	go.starlark.net v0.0.0-20220203230714-bb14e151c28f // indirect
-	golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab // indirect
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/cli-runtime v0.23.3 // indirect
-	k8s.io/component-base v0.23.3 // indirect
-	k8s.io/klog/v2 v2.40.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf // indirect
-	k8s.io/kubectl v0.23.3 // indirect
-	k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect
-	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
-	sigs.k8s.io/kustomize/api v0.11.1 // indirect
-	sigs.k8s.io/kustomize/kyaml v0.13.3 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	golang.org/x/text v0.3.5 // indirect
+	k8s.io/api v0.21.2
+	k8s.io/apimachinery v0.21.2
+	k8s.io/client-go v0.21.2
 )
 
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
@@ -141,11 +36,3 @@ replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 replace github.com/up9inc/mizu/tap v0.0.0 => ../tap
 
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api
-
-replace github.com/up9inc/mizu/tap/extensions/amqp v0.0.0 => ../tap/extensions/amqp
-
-replace github.com/up9inc/mizu/tap/extensions/http v0.0.0 => ../tap/extensions/http
-
-replace github.com/up9inc/mizu/tap/extensions/kafka v0.0.0 => ../tap/extensions/kafka
-
-replace github.com/up9inc/mizu/tap/extensions/redis v0.0.0 => ../tap/extensions/redis

agent/kratos/Dockerfile

@@ -0,0 +1,14 @@
+FROM gcr.io/up9-docker-hub/mizu-kratos-base/simple-password-policy:latest
+
+USER root
+
+RUN apk add sqlite
+
+RUN mkdir -p /etc/config/kratos
+
+COPY ./kratos.yml /etc/config/kratos/kratos.yml
+COPY ./identity.schema.json /etc/config/kratos/identity.schema.json
+COPY ./start.sh /opt/start.sh
+RUN chmod +x /opt/start.sh
+
+ENTRYPOINT ["/opt/start.sh"]

agent/kratos/build-push-featurebranch.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+
+GCP_PROJECT=up9-docker-hub
+REPOSITORY=gcr.io/$GCP_PROJECT
+SERVER_NAME=mizu-kratos
+GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
+
+DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
+SEM_VER=${SEM_VER=0.0.0}
+ARCH=amd64
+
+DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
+
+if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
+then
+  echo "Pushing to $GIT_BRANCH is allowed only via CI"
+  exit 1
+fi
+
+echo "building ${DOCKER_TAGGED_BUILDS[@]}"
+DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
+docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
+
+for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
+do
+  echo pushing "$DOCKER_TAG"
+  docker push "$DOCKER_TAG"
+done

agent/kratos/identity.schema.json

@@ -0,0 +1,43 @@
+{
+	"$id": "https://schemas.ory.sh/presets/kratos/quickstart/email-password/identity.schema.json",
+	"$schema": "http://json-schema.org/draft-07/schema#",
+	"title": "Person",
+	"type": "object",
+	"properties": {
+		"traits": {
+			"type": "object",
+			"properties": {
+				"username": {
+					"type": "string",
+					"format": "username",
+					"title": "Username",
+					"minLength": 3,
+					"ory.sh/kratos": {
+						"credentials": {
+							"password": {
+								"identifier": true
+							}
+						}
+					}
+				},
+				"name": {
+					"type": "object",
+					"properties": {
+						"first": {
+							"title": "First Name",
+							"type": "string"
+						},
+						"last": {
+							"title": "Last Name",
+							"type": "string"
+						}
+					}
+				}
+			},
+			"required": [
+				"username"
+			],
+			"additionalProperties": false
+		}
+	}
+}

agent/kratos/kratos.yml

@@ -0,0 +1,84 @@
+version: v0.8.2-alpha.1
+
+dsn: sqlite:///app/data/kratos.sqlite?_fk=true
+
+serve:
+  public:
+    base_url: http://127.0.0.1:4433/
+    cors:
+      enabled: true
+  admin:
+    base_url: http://kratos:4434/
+
+selfservice:
+  default_browser_return_url: http://127.0.0.1:4455/
+  whitelisted_return_urls:
+    - http://127.0.0.1:4455
+
+  methods:
+    password:
+      enabled: true
+
+  flows:
+    error:
+      ui_url: http://127.0.0.1:4455/error
+
+    settings:
+      ui_url: http://127.0.0.1:4455/settings
+      privileged_session_max_age: 15m
+
+    recovery:
+      enabled: true
+      ui_url: http://127.0.0.1:4455/recovery
+
+    verification:
+      enabled: false
+      ui_url: http://127.0.0.1:4455/verification
+      after:
+        default_browser_return_url: http://127.0.0.1:4455/
+
+    logout:
+      after:
+        default_browser_return_url: http://127.0.0.1:4455/login
+
+    login:
+      ui_url: http://127.0.0.1:4455/login
+      lifespan: 10m
+
+    registration:
+      lifespan: 10m
+      ui_url: http://127.0.0.1:4455/registration
+      after:
+        password:
+          hooks:
+            -
+              hook: session
+
+log:
+  level: info
+  format: text
+  leak_sensitive_values: true
+
+secrets:
+  cookie:
+    - PLEASE-CHANGE-ME-I-AM-VERY-INSECURE
+  cipher:
+    - 32-LONG-SECRET-NOT-SECURE-AT-ALL
+
+ciphers:
+  algorithm: xchacha20-poly1305
+
+hashers:
+  argon2:
+    parallelism: 1
+    memory: 128MB
+    iterations: 2
+    salt_length: 16
+    key_length: 16
+
+identity:
+  default_schema_url: file:///etc/config/kratos/identity.schema.json
+
+courier:
+  smtp:
+    connection_uri: smtps://test:test@mailslurper:1025/?skip_ssl_verify=true

agent/kratos/start.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+kratos migrate sql sqlite:///app/data/kratos.sqlite?_fk=true --yes # this initializes the db
+kratos serve -c /etc/config/kratos/kratos.yml --watch-courier # start kratos

agent/main.go

@@ -6,33 +6,36 @@ import (
 	"flag"
 	"fmt"
 	"io/ioutil"
+	"mizuserver/pkg/api"
+	"mizuserver/pkg/config"
+	"mizuserver/pkg/controllers"
+	"mizuserver/pkg/middlewares"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/oas"
+	"mizuserver/pkg/routes"
+	"mizuserver/pkg/servicemap"
+	"mizuserver/pkg/up9"
+	"mizuserver/pkg/utils"
 	"net/http"
 	"os"
 	"os/signal"
+	"path"
+	"path/filepath"
+	"plugin"
+	"sort"
 	"strconv"
 	"strings"
 	"syscall"
 	"time"
 
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/antelman107/net-wait-go/wait"
 	"github.com/gin-contrib/static"
 	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/agent/pkg/dependency"
-	"github.com/up9inc/mizu/agent/pkg/elastic"
-	"github.com/up9inc/mizu/agent/pkg/entries"
-	"github.com/up9inc/mizu/agent/pkg/middlewares"
-	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/agent/pkg/oas"
-	"github.com/up9inc/mizu/agent/pkg/routes"
-	"github.com/up9inc/mizu/agent/pkg/servicemap"
-	"github.com/up9inc/mizu/agent/pkg/up9"
-	"github.com/up9inc/mizu/agent/pkg/utils"
-
-	"github.com/up9inc/mizu/agent/pkg/api"
-	"github.com/up9inc/mizu/agent/pkg/app"
-	"github.com/up9inc/mizu/agent/pkg/config"
-
 	"github.com/gorilla/websocket"
 	"github.com/op/go-logging"
+	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap"
@@ -47,32 +50,97 @@ var namespace = flag.String("namespace", "", "Resolve IPs if they belong to reso
 var harsReaderMode = flag.Bool("hars-read", false, "Run in hars-read mode")
 var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
 
+var extensions []*tapApi.Extension             // global
+var extensionsMap map[string]*tapApi.Extension // global
+
+var startTime int64
+
 const (
 	socketConnectionRetries    = 30
 	socketConnectionRetryDelay = time.Second * 2
 	socketHandshakeTimeout     = time.Second * 2
+	uiIndexPath                = "./site/index.html"
 )
 
 func main() {
-	initializeDependencies()
 	logLevel := determineLogLevel()
-	logger.InitLoggerStd(logLevel)
+	logger.InitLoggerStderrOnly(logLevel)
 	flag.Parse()
-
-	app.LoadExtensions()
+	if err := config.LoadConfig(); err != nil {
+		logger.Log.Fatalf("Error loading config file %v", err)
+	}
+	loadExtensions()
 
 	if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode {
 		panic("One of the flags --tap, --api or --standalone or --hars-read must be provided")
 	}
 
 	if *standaloneMode {
-		runInStandaloneMode()
+		api.StartResolving(*namespace)
+
+		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
+		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+
+		filteringOptions := getTrafficFilteringOptions()
+		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+		tapOpts := &tap.TapOpts{HostMode: hostMode}
+		tap.StartPassiveTapper(tapOpts, outputItemsChannel, extensions, filteringOptions)
+
+		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
+		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
+
+		hostApi(nil)
 	} else if *tapperMode {
-		runInTapperMode()
+		logger.Log.Infof("Starting tapper, websocket address: %s", *apiServerAddress)
+		if *apiServerAddress == "" {
+			panic("API server address must be provided with --api-server-address when using --tap")
+		}
+
+		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+		tapOpts := &tap.TapOpts{HostMode: hostMode}
+		tapTargets := getTapTargets()
+		if tapTargets != nil {
+			tapOpts.FilterAuthorities = tapTargets
+			logger.Log.Infof("Filtering for the following authorities: %v", tapOpts.FilterAuthorities)
+		}
+
+		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+
+		filteringOptions := getTrafficFilteringOptions()
+		tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, extensions, filteringOptions)
+		socketConnection, err := dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay)
+		if err != nil {
+			panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
+		}
+		logger.Log.Infof("Connected successfully to websocket %s", *apiServerAddress)
+
+		go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
 	} else if *apiServerMode {
-		utils.StartServer(runInApiServerMode(*namespace))
+		configureBasenineServer(shared.BasenineHost, shared.BaseninePort)
+		startTime = time.Now().UnixNano() / int64(time.Millisecond)
+		api.StartResolving(*namespace)
+
+		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
+		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+		enableExpFeatureIfNeeded()
+		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
+		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
+
+		syncEntriesConfig := getSyncEntriesConfig()
+		if syncEntriesConfig != nil {
+			if err := up9.SyncEntries(syncEntriesConfig); err != nil {
+				logger.Log.Error("Error syncing entries, err: %v", err)
+			}
+		}
+
+		hostApi(outputItemsChannel)
 	} else if *harsReaderMode {
-		runInHarReaderMode()
+		outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000)
+		filteredHarChannel := make(chan *tapApi.OutputChannelItem)
+
+		go filterItems(outputItemsChannel, filteredHarChannel)
+		go api.StartReadingEntries(filteredHarChannel, harsDir, extensionsMap)
+		hostApi(nil)
 	}
 
 	signalChan := make(chan os.Signal, 1)
@@ -82,38 +150,117 @@ func main() {
 	logger.Log.Info("Exiting")
 }
 
-func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) *gin.Engine {
+func enableExpFeatureIfNeeded() {
+	if config.Config.OAS {
+		oas.GetOasGeneratorInstance().Start()
+	}
+	if config.Config.ServiceMap {
+		servicemap.GetInstance().SetConfig(config.Config)
+	}
+}
+
+func configureBasenineServer(host string, port string) {
+	if !wait.New(
+		wait.WithProto("tcp"),
+		wait.WithWait(200*time.Millisecond),
+		wait.WithBreak(50*time.Millisecond),
+		wait.WithDeadline(5*time.Second),
+		wait.WithDebug(true),
+	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
+		logger.Log.Panicf("Basenine is not available!")
+	}
+
+	// Limit the database size to default 200MB
+	err := basenine.Limit(host, port, config.Config.MaxDBSizeBytes)
+	if err != nil {
+		logger.Log.Panicf("Error while limiting database size: %v", err)
+	}
+
+	// Define the macros
+	for _, extension := range extensions {
+		macros := extension.Dissector.Macros()
+		for macro, expanded := range macros {
+			err = basenine.Macro(host, port, macro, expanded)
+			if err != nil {
+				logger.Log.Panicf("Error while adding a macro: %v", err)
+			}
+		}
+	}
+}
+
+func loadExtensions() {
+	dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
+	extensionsDir := path.Join(dir, "./extensions/")
+
+	files, err := ioutil.ReadDir(extensionsDir)
+	if err != nil {
+		logger.Log.Fatal(err)
+	}
+	extensions = make([]*tapApi.Extension, len(files))
+	extensionsMap = make(map[string]*tapApi.Extension)
+	for i, file := range files {
+		filename := file.Name()
+		logger.Log.Infof("Loading extension: %s", filename)
+		extension := &tapApi.Extension{
+			Path: path.Join(extensionsDir, filename),
+		}
+		plug, _ := plugin.Open(extension.Path)
+		extension.Plug = plug
+		symDissector, err := plug.Lookup("Dissector")
+
+		var dissector tapApi.Dissector
+		var ok bool
+		dissector, ok = symDissector.(tapApi.Dissector)
+		if err != nil || !ok {
+			panic(fmt.Sprintf("Failed to load the extension: %s", extension.Path))
+		}
+		dissector.Register(extension)
+		extension.Dissector = dissector
+		extensions[i] = extension
+		extensionsMap[extension.Protocol.Name] = extension
+	}
+
+	sort.Slice(extensions, func(i, j int) bool {
+		return extensions[i].Protocol.Priority < extensions[j].Protocol.Priority
+	})
+
+	for _, extension := range extensions {
+		logger.Log.Infof("Extension Properties: %+v", extension)
+	}
+
+	controllers.InitExtensionsMap(extensionsMap)
+}
+
+func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 	app := gin.Default()
 
 	app.GET("/echo", func(c *gin.Context) {
-		c.JSON(http.StatusOK, "Here is Mizu agent")
+		c.String(http.StatusOK, "Here is Mizu agent")
 	})
 
 	eventHandlers := api.RoutesEventHandlers{
 		SocketOutChannel: socketHarOutputChannel,
 	}
 
-	app.Use(disableRootStaticCache())
+	app.Use(DisableRootStaticCache())
 
-	staticFolder := "./site"
-	indexStaticFile := staticFolder + "/index.html"
-	if err := setUIFlags(indexStaticFile); err != nil {
-		logger.Log.Errorf("Error setting ui flags, err: %v", err)
+	if err := setUIFlags(); err != nil {
+		logger.Log.Errorf("Error setting ui mode, err: %v", err)
 	}
-
-	app.Use(static.ServeRoot("/", staticFolder))
-	app.NoRoute(func(c *gin.Context) {
-		c.File(indexStaticFile)
-	})
+	app.Use(static.ServeRoot("/", "./site"))
 
 	app.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
 
-	api.WebSocketRoutes(app, &eventHandlers)
+	api.WebSocketRoutes(app, &eventHandlers, startTime)
 
+	if config.Config.StandaloneMode {
+		routes.ConfigRoutes(app)
+		routes.UserRoutes(app)
+		routes.InstallRoutes(app)
+	}
 	if config.Config.OAS {
 		routes.OASRoutes(app)
 	}
-
 	if config.Config.ServiceMap {
 		routes.ServiceMapRoutes(app)
 	}
@@ -122,108 +269,11 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) *gin.Engin
 	routes.EntriesRoutes(app)
 	routes.MetadataRoutes(app)
 	routes.StatusRoutes(app)
-	routes.DbRoutes(app)
-
-	return app
+	routes.NotFoundRoute(app)
+	utils.StartServer(app)
 }
 
-func runInApiServerMode(namespace string) *gin.Engine {
-	if err := config.LoadConfig(); err != nil {
-		logger.Log.Fatalf("Error loading config file %v", err)
-	}
-	app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort, config.Config.MaxDBSizeBytes, config.Config.LogLevel, config.Config.InsertionFilter)
-	api.StartResolving(namespace)
-
-	enableExpFeatureIfNeeded()
-
-	syncEntriesConfig := getSyncEntriesConfig()
-	if syncEntriesConfig != nil {
-		if err := up9.SyncEntries(syncEntriesConfig); err != nil {
-			logger.Log.Error("Error syncing entries, err: %v", err)
-		}
-	}
-
-	return hostApi(app.GetEntryInputChannel())
-}
-
-func runInTapperMode() {
-	logger.Log.Infof("Starting tapper, websocket address: %s", *apiServerAddress)
-	if *apiServerAddress == "" {
-		panic("API server address must be provided with --api-server-address when using --tap")
-	}
-
-	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
-	tapOpts := &tap.TapOpts{HostMode: hostMode}
-
-	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-
-	filteringOptions := getTrafficFilteringOptions()
-	tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, app.Extensions, filteringOptions)
-	socketConnection, err := dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay)
-	if err != nil {
-		panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
-	}
-	logger.Log.Infof("Connected successfully to websocket %s", *apiServerAddress)
-
-	go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
-}
-
-func runInStandaloneMode() {
-	api.StartResolving(*namespace)
-
-	outputItemsChannel := make(chan *tapApi.OutputChannelItem)
-	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-
-	filteringOptions := getTrafficFilteringOptions()
-	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
-	tapOpts := &tap.TapOpts{HostMode: hostMode}
-	tap.StartPassiveTapper(tapOpts, outputItemsChannel, app.Extensions, filteringOptions)
-
-	go app.FilterItems(outputItemsChannel, filteredOutputItemsChannel)
-	go api.StartReadingEntries(filteredOutputItemsChannel, nil, app.ExtensionsMap)
-
-	ginApp := hostApi(nil)
-	utils.StartServer(ginApp)
-}
-
-func runInHarReaderMode() {
-	outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000)
-	filteredHarChannel := make(chan *tapApi.OutputChannelItem)
-
-	go app.FilterItems(outputItemsChannel, filteredHarChannel)
-	go api.StartReadingEntries(filteredHarChannel, harsDir, app.ExtensionsMap)
-	ginApp := hostApi(nil)
-	utils.StartServer(ginApp)
-}
-
-func enableExpFeatureIfNeeded() {
-	if config.Config.OAS {
-		oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
-		oasGenerator.Start(nil)
-	}
-	if config.Config.ServiceMap {
-		serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMap)
-		serviceMapGenerator.Enable()
-	}
-	elastic.GetInstance().Configure(config.Config.Elastic)
-}
-
-func getSyncEntriesConfig() *shared.SyncEntriesConfig {
-	syncEntriesConfigJson := os.Getenv(shared.SyncEntriesConfigEnvVar)
-	if syncEntriesConfigJson == "" {
-		return nil
-	}
-
-	var syncEntriesConfig = &shared.SyncEntriesConfig{}
-	err := json.Unmarshal([]byte(syncEntriesConfigJson), syncEntriesConfig)
-	if err != nil {
-		panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.SyncEntriesConfig struct, err: %v", shared.SyncEntriesConfigEnvVar, syncEntriesConfigJson, err))
-	}
-
-	return syncEntriesConfig
-}
-
-func disableRootStaticCache() gin.HandlerFunc {
+func DisableRootStaticCache() gin.HandlerFunc {
 	return func(c *gin.Context) {
 		if c.Request.RequestURI == "/" {
 			// Disable cache only for the main static route
@@ -234,13 +284,14 @@ func disableRootStaticCache() gin.HandlerFunc {
 	}
 }
 
-func setUIFlags(uiIndexPath string) error {
+func setUIFlags() error {
 	read, err := ioutil.ReadFile(uiIndexPath)
 	if err != nil {
 		return err
 	}
 
-	replacedContent := strings.Replace(string(read), "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS), 1)
+	replacedContent := strings.Replace(string(read), "__IS_STANDALONE__", strconv.FormatBool(config.Config.StandaloneMode), 1)
+	replacedContent = strings.Replace(replacedContent, "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS), 1)
 	replacedContent = strings.Replace(replacedContent, "__IS_SERVICE_MAP_ENABLED__", strconv.FormatBool(config.Config.ServiceMap), 1)
 
 	err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0)
@@ -251,6 +302,28 @@ func setUIFlags(uiIndexPath string) error {
 	return nil
 }
 
+func parseEnvVar(env string) map[string][]v1.Pod {
+	var mapOfList map[string][]v1.Pod
+
+	val, present := os.LookupEnv(env)
+
+	if !present {
+		return mapOfList
+	}
+
+	err := json.Unmarshal([]byte(val), &mapOfList)
+	if err != nil {
+		panic(fmt.Sprintf("env var %s's value of %v is invalid! must be map[string][]v1.Pod %v", env, mapOfList, err))
+	}
+	return mapOfList
+}
+
+func getTapTargets() []v1.Pod {
+	nodeName := os.Getenv(shared.NodeNameEnvVar)
+	tappedAddressesPerNodeDict := parseEnvVar(shared.TappedAddressesPerNodeDictEnvVar)
+	return tappedAddressesPerNodeDict[nodeName]
+}
+
 func getTrafficFilteringOptions() *tapApi.TrafficFilteringOptions {
 	filteringOptionsJson := os.Getenv(shared.MizuFilteringOptionsEnvVar)
 	if filteringOptionsJson == "" {
@@ -267,6 +340,16 @@ func getTrafficFilteringOptions() *tapApi.TrafficFilteringOptions {
 	return &filteringOptions
 }
 
+func filterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem) {
+	for message := range inChannel {
+		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
+			continue
+		}
+
+		outChannel <- message
+	}
+}
+
 func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tapApi.OutputChannelItem) {
 	if connection == nil {
 		panic("Websocket connection is nil")
@@ -302,6 +385,21 @@ func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-cha
 	}
 }
 
+func getSyncEntriesConfig() *shared.SyncEntriesConfig {
+	syncEntriesConfigJson := os.Getenv(shared.SyncEntriesConfigEnvVar)
+	if syncEntriesConfigJson == "" {
+		return nil
+	}
+
+	var syncEntriesConfig = &shared.SyncEntriesConfig{}
+	err := json.Unmarshal([]byte(syncEntriesConfigJson), syncEntriesConfig)
+	if err != nil {
+		panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.SyncEntriesConfig struct, err: %v", shared.SyncEntriesConfigEnvVar, syncEntriesConfigJson, err))
+	}
+
+	return syncEntriesConfig
+}
+
 func determineLogLevel() (logLevel logging.Level) {
 	logLevel, err := logging.LogLevel(os.Getenv(shared.LogLevelEnvVar))
 	if err != nil {
@@ -353,14 +451,6 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
 					} else {
 						tap.UpdateTapTargets(tapConfigMessage.TapTargets)
 					}
-				case shared.WebSocketMessageTypeUpdateTappedPods:
-					var tappedPodsMessage shared.WebSocketTappedPodsMessage
-					if err := json.Unmarshal(message, &tappedPodsMessage); err != nil {
-						logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
-						return
-					}
-					nodeName := os.Getenv(shared.NodeNameEnvVar)
-					tap.UpdateTapTargets(tappedPodsMessage.NodeToTappedPodMap[nodeName])
 				default:
 					logger.Log.Warningf("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
 				}
@@ -368,11 +458,3 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
 		}
 	}
 }
-
-func initializeDependencies() {
-	dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
-	dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} { return oas.GetDefaultOasGeneratorInstance() })
-	dependency.RegisterGenerator(dependency.EntriesProvider, func() interface{} { return &entries.BasenineEntriesProvider{} })
-	dependency.RegisterGenerator(dependency.EntriesSocketStreamer, func() interface{} { return &api.BasenineEntryStreamer{} })
-	dependency.RegisterGenerator(dependency.EntryStreamerSocketConnector, func() interface{} { return &api.DefaultEntryStreamerSocketConnector{} })
-}

agent/pkg/api/entry_streamer_socket_connector.go

@@ -1,57 +0,0 @@
-package api
-
-import (
-	"fmt"
-
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/shared/logger"
-	tapApi "github.com/up9inc/mizu/tap/api"
-)
-
-type EntryStreamerSocketConnector interface {
-	SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams)
-	SendMetadata(socketId int, metadata *basenine.Metadata)
-	SendToastError(socketId int, err error)
-	CleanupSocket(socketId int)
-}
-
-type DefaultEntryStreamerSocketConnector struct{}
-
-func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) {
-	var message []byte
-	if params.EnableFullEntries {
-		message, _ = models.CreateFullEntryWebSocketMessage(entry)
-	} else {
-		extension := extensionsMap[entry.Protocol.Name]
-		base := extension.Dissector.Summarize(entry)
-		message, _ = models.CreateBaseEntryWebSocketMessage(base)
-	}
-
-	if err := SendToSocket(socketId, message); err != nil {
-		logger.Log.Error(err)
-	}
-}
-
-func (e *DefaultEntryStreamerSocketConnector) SendMetadata(socketId int, metadata *basenine.Metadata) {
-	metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
-	if err := SendToSocket(socketId, metadataBytes); err != nil {
-		logger.Log.Error(err)
-	}
-}
-
-func (e *DefaultEntryStreamerSocketConnector) SendToastError(socketId int, err error) {
-	toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
-		Type:      "error",
-		AutoClose: 5000,
-		Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
-	})
-	if err := SendToSocket(socketId, toastBytes); err != nil {
-		logger.Log.Error(err)
-	}
-}
-
-func (e *DefaultEntryStreamerSocketConnector) CleanupSocket(socketId int) {
-	socketObj := connectedWebsockets[socketId]
-	socketCleanup(socketId, socketObj)
-}

agent/pkg/api/main.go

@@ -5,29 +5,26 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"mizuserver/pkg/holder"
+	"mizuserver/pkg/providers"
 	"os"
 	"path"
 	"sort"
 	"strings"
 	"time"
 
-	"github.com/up9inc/mizu/agent/pkg/models"
-
-	"github.com/up9inc/mizu/agent/pkg/dependency"
-	"github.com/up9inc/mizu/agent/pkg/elastic"
-	"github.com/up9inc/mizu/agent/pkg/har"
-	"github.com/up9inc/mizu/agent/pkg/holder"
-	"github.com/up9inc/mizu/agent/pkg/providers"
-
-	"github.com/up9inc/mizu/agent/pkg/servicemap"
-
-	"github.com/up9inc/mizu/agent/pkg/resolver"
-	"github.com/up9inc/mizu/agent/pkg/utils"
+	"mizuserver/pkg/servicemap"
 
+	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/oas"
+	"mizuserver/pkg/resolver"
+	"mizuserver/pkg/utils"
+
 	basenine "github.com/up9inc/basenine/client/go"
 )
 
@@ -44,8 +41,10 @@ func StartResolving(namespace string) {
 	res.Start(ctx)
 	go func() {
 		for {
-			err := <-errOut
-			logger.Log.Infof("name resolving error %s", err)
+			select {
+			case err := <-errOut:
+				logger.Log.Infof("name resolving error %s", err)
+			}
 		}
 	}()
 
@@ -67,7 +66,7 @@ func startReadingFiles(workingDir string) {
 		return
 	}
 
-	for {
+	for true {
 		dir, _ := os.Open(workingDir)
 		dirFiles, _ := dir.Readdir(-1)
 
@@ -105,11 +104,9 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 
 	connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
 	if err != nil {
-		logger.Log.Panicf("Can't establish a new connection to Basenine server: %v", err)
-	}
-	if err = connection.InsertMode(); err != nil {
-		logger.Log.Panicf("Insert mode call failed: %v", err)
+		panic(err)
 	}
+	connection.InsertMode()
 
 	disableOASValidation := false
 	ctx := context.Background()
@@ -120,15 +117,15 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 	}
 
 	for item := range outputItems {
+		providers.EntryAdded()
+
 		extension := extensionsMap[item.Protocol.Name]
-		resolvedSource, resolvedDestionation, namespace := resolveIP(item.ConnectionInfo)
-		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation, namespace)
+		resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
+		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation)
 		if extension.Protocol.Name == "http" {
 			if !disableOASValidation {
 				var httpPair tapApi.HTTPRequestResponsePair
-				if err := json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair); err != nil {
-					logger.Log.Error(err)
-				}
+				json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair)
 
 				contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
 				mizuEntry.ContractStatus = contract.Status
@@ -137,62 +134,45 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 				mizuEntry.ContractContent = contract.Content
 			}
 
-			harEntry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
+			harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
 			if err == nil {
 				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name)
 				mizuEntry.Rules = rules
 			}
+
+			oas.GetOasGeneratorInstance().PushEntry(harEntry)
 		}
 
 		data, err := json.Marshal(mizuEntry)
 		if err != nil {
 			panic(err)
 		}
+		connection.SendText(string(data))
 
-		providers.EntryAdded(len(data))
-
-		if err = connection.SendText(string(data)); err != nil {
-			logger.Log.Panicf("An error occured while inserting a new record to database: %v", err)
-		}
-
-		serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMapSink)
-		serviceMapGenerator.NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
-
-		elastic.GetInstance().PushEntry(mizuEntry)
+		servicemap.GetInstance().NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
 	}
 }
 
-func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, resolvedDestination string, namespace string) {
+func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, resolvedDestination string) {
 	if k8sResolver != nil {
 		unresolvedSource := connectionInfo.ClientIP
-		resolvedSourceObject := k8sResolver.Resolve(unresolvedSource)
-		if resolvedSourceObject == nil {
+		resolvedSource = k8sResolver.Resolve(unresolvedSource)
+		if resolvedSource == "" {
 			logger.Log.Debugf("Cannot find resolved name to source: %s", unresolvedSource)
 			if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" {
 				return
 			}
-		} else {
-			resolvedSource = resolvedSourceObject.FullAddress
-			namespace = resolvedSourceObject.Namespace
 		}
-
 		unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort)
-		resolvedDestinationObject := k8sResolver.Resolve(unresolvedDestination)
-		if resolvedDestinationObject == nil {
+		resolvedDestination = k8sResolver.Resolve(unresolvedDestination)
+		if resolvedDestination == "" {
 			logger.Log.Debugf("Cannot find resolved name to dest: %s", unresolvedDestination)
 			if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" {
 				return
 			}
-		} else {
-			resolvedDestination = resolvedDestinationObject.FullAddress
-			// Overwrite namespace (if it was set according to the source)
-			// Only overwrite if non-empty
-			if resolvedDestinationObject.Namespace != "" {
-				namespace = resolvedDestinationObject.Namespace
-			}
 		}
 	}
-	return resolvedSource, resolvedDestination, namespace
+	return resolvedSource, resolvedDestination
 }
 
 func CheckIsServiceIP(address string) bool {

agent/pkg/api/socket_data_streamer.go

@@ -1,94 +0,0 @@
-package api
-
-import (
-	"context"
-	"encoding/json"
-
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/agent/pkg/dependency"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	tapApi "github.com/up9inc/mizu/tap/api"
-)
-
-type EntryStreamer interface {
-	Get(ctx context.Context, socketId int, params *WebSocketParams) error
-}
-
-type BasenineEntryStreamer struct{}
-
-func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *WebSocketParams) error {
-	var connection *basenine.Connection
-
-	entryStreamerSocketConnector := dependency.GetInstance(dependency.EntryStreamerSocketConnector).(EntryStreamerSocketConnector)
-
-	connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
-	if err != nil {
-		logger.Log.Errorf("failed to establish a connection to Basenine: %v", err)
-		entryStreamerSocketConnector.CleanupSocket(socketId)
-		return err
-	}
-
-	data := make(chan []byte)
-	meta := make(chan []byte)
-
-	query := params.Query
-	err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
-	if err != nil {
-		entryStreamerSocketConnector.SendToastError(socketId, err)
-	}
-
-	handleDataChannel := func(c *basenine.Connection, data chan []byte) {
-		for {
-			bytes := <-data
-
-			if string(bytes) == basenine.CloseChannel {
-				return
-			}
-
-			var entry *tapApi.Entry
-			err = json.Unmarshal(bytes, &entry)
-			if err != nil {
-				logger.Log.Debugf("Error unmarshalling entry: %v", err.Error())
-				continue
-			}
-
-			entryStreamerSocketConnector.SendEntry(socketId, entry, params)
-		}
-	}
-
-	handleMetaChannel := func(c *basenine.Connection, meta chan []byte) {
-		for {
-			bytes := <-meta
-
-			if string(bytes) == basenine.CloseChannel {
-				return
-			}
-
-			var metadata *basenine.Metadata
-			err = json.Unmarshal(bytes, &metadata)
-			if err != nil {
-				logger.Log.Debugf("Error unmarshalling metadata: %v", err.Error())
-				continue
-			}
-
-			entryStreamerSocketConnector.SendMetadata(socketId, metadata)
-		}
-	}
-
-	go handleDataChannel(connection, data)
-	go handleMetaChannel(connection, meta)
-
-	if err = connection.Query(query, data, meta); err != nil {
-		logger.Log.Panicf("Query mode call failed: %v", err)
-	}
-
-	go func() {
-		<-ctx.Done()
-		data <- []byte(basenine.CloseChannel)
-		meta <- []byte(basenine.CloseChannel)
-		connection.Close()
-	}()
-
-	return nil
-}

agent/pkg/api/socket_routes.go

@@ -1,29 +1,28 @@
 package api
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
+	"mizuserver/pkg/middlewares"
+	"mizuserver/pkg/models"
 	"net/http"
 	"sync"
 	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
-	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/agent/pkg/utils"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
-var extensionsMap map[string]*tapApi.Extension // global
-
-func InitExtensionsMap(ref map[string]*tapApi.Extension) {
-	extensionsMap = ref
-}
-
 type EventHandlers interface {
-	WebSocketConnect(c *gin.Context, socketId int, isTapper bool)
+	WebSocketConnect(socketId int, isTapper bool)
 	WebSocketDisconnect(socketId int, isTapper bool)
-	WebSocketMessage(socketId int, isTapper bool, message []byte)
+	WebSocketMessage(socketId int, message []byte)
 }
 
 type SocketConnection struct {
@@ -33,51 +32,34 @@ type SocketConnection struct {
 	isTapper      bool
 }
 
-type WebSocketParams struct {
-	Query             string `json:"query"`
-	EnableFullEntries bool   `json:"enableFullEntries"`
+var websocketUpgrader = websocket.Upgrader{
+	ReadBufferSize:  1024,
+	WriteBufferSize: 1024,
 }
 
-var (
-	websocketUpgrader = websocket.Upgrader{
-		ReadBufferSize:  1024,
-		WriteBufferSize: 1024,
-	}
-
-	websocketIdsLock            = sync.Mutex{}
-	connectedWebsockets         map[int]*SocketConnection
-	connectedWebsocketIdCounter = 0
-	SocketGetBrowserHandler     gin.HandlerFunc
-	SocketGetTapperHandler      gin.HandlerFunc
-)
+var websocketIdsLock = sync.Mutex{}
+var connectedWebsockets map[int]*SocketConnection
+var connectedWebsocketIdCounter = 0
 
 func init() {
 	websocketUpgrader.CheckOrigin = func(r *http.Request) bool { return true } // like cors for web socket
-	connectedWebsockets = make(map[int]*SocketConnection)
+	connectedWebsockets = make(map[int]*SocketConnection, 0)
 }
 
-func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
-	SocketGetBrowserHandler = func(c *gin.Context) {
-		websocketHandler(c, eventHandlers, false)
-	}
-
-	SocketGetTapperHandler = func(c *gin.Context) {
-		websocketHandler(c, eventHandlers, true)
-	}
-
+func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) {
 	app.GET("/ws", func(c *gin.Context) {
-		SocketGetBrowserHandler(c)
-	})
+		websocketHandler(c.Writer, c.Request, eventHandlers, false, startTime)
+	}, middlewares.RequiresAuth())
 
 	app.GET("/wsTapper", func(c *gin.Context) { // TODO: add m2m authentication to this route
-		SocketGetTapperHandler(c)
+		websocketHandler(c.Writer, c.Request, eventHandlers, true, startTime)
 	})
 }
 
-func websocketHandler(c *gin.Context, eventHandlers EventHandlers, isTapper bool) {
-	ws, err := websocketUpgrader.Upgrade(c.Writer, c.Request, nil)
+func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool, startTime int64) {
+	ws, err := websocketUpgrader.Upgrade(w, r, nil)
 	if err != nil {
-		logger.Log.Errorf("failed to set websocket upgrade: %v", err)
+		logger.Log.Errorf("Failed to set websocket upgrade: %v", err)
 		return
 	}
 
@@ -89,63 +71,110 @@ func websocketHandler(c *gin.Context, eventHandlers EventHandlers, isTapper bool
 
 	websocketIdsLock.Unlock()
 
+	var connection *basenine.Connection
+	var isQuerySet bool
+
+	// `!isTapper` means it's a connection from the web UI
+	if !isTapper {
+		connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	data := make(chan []byte)
+	meta := make(chan []byte)
+
 	defer func() {
 		socketCleanup(socketId, connectedWebsockets[socketId])
+		data <- []byte(basenine.CloseChannel)
+		meta <- []byte(basenine.CloseChannel)
+		connection.Close()
 	}()
 
-	eventHandlers.WebSocketConnect(c, socketId, isTapper)
+	eventHandlers.WebSocketConnect(socketId, isTapper)
 
-	startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(utils.StartTime)
-
-	if err = SendToSocket(socketId, startTimeBytes); err != nil {
-		logger.Log.Error(err)
-	}
+	startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(startTime)
+	SendToSocket(socketId, startTimeBytes)
 
 	for {
 		_, msg, err := ws.ReadMessage()
 		if err != nil {
 			if _, ok := err.(*websocket.CloseError); ok {
-				logger.Log.Debugf("received websocket close message, socket id: %d", socketId)
+				logger.Log.Debugf("Received websocket close message, socket id: %d", socketId)
 			} else {
-				logger.Log.Errorf("error reading message, socket id: %d, error: %v", socketId, err)
+				logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
 			}
 
 			break
 		}
 
-		eventHandlers.WebSocketMessage(socketId, isTapper, msg)
-	}
-}
+		if !isTapper && !isQuerySet {
+			query := string(msg)
+			err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
+			if err != nil {
+				toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
+					Type:      "error",
+					AutoClose: 5000,
+					Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
+				})
+				SendToSocket(socketId, toastBytes)
+				break
+			}
 
-func SendToSocket(socketId int, message []byte) error {
-	socketObj := connectedWebsockets[socketId]
-	if socketObj == nil {
-		return fmt.Errorf("socket %v is disconnected", socketId)
-	}
+			isQuerySet = true
 
-	var sent = false
-	time.AfterFunc(time.Second*5, func() {
-		if !sent {
-			logger.Log.Error("socket timed out")
-			socketCleanup(socketId, socketObj)
+			handleDataChannel := func(c *basenine.Connection, data chan []byte) {
+				for {
+					bytes := <-data
+
+					if string(bytes) == basenine.CloseChannel {
+						return
+					}
+
+					var entry *tapApi.Entry
+					err = json.Unmarshal(bytes, &entry)
+
+					base := tapApi.Summarize(entry)
+
+					baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
+					SendToSocket(socketId, baseEntryBytes)
+				}
+			}
+
+			handleMetaChannel := func(c *basenine.Connection, meta chan []byte) {
+				for {
+					bytes := <-meta
+
+					if string(bytes) == basenine.CloseChannel {
+						return
+					}
+
+					var metadata *basenine.Metadata
+					err = json.Unmarshal(bytes, &metadata)
+					if err != nil {
+						logger.Log.Debugf("Error recieving metadata: %v", err.Error())
+					}
+
+					metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
+					SendToSocket(socketId, metadataBytes)
+				}
+			}
+
+			go handleDataChannel(connection, data)
+			go handleMetaChannel(connection, meta)
+
+			connection.Query(query, data, meta)
+		} else {
+			eventHandlers.WebSocketMessage(socketId, msg)
 		}
-	})
-
-	socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
-	err := socketObj.connection.WriteMessage(1, message)
-	socketObj.lock.Unlock()
-	sent = true
-
-	if err != nil {
-		return fmt.Errorf("failed to write message to socket %v, err: %w", socketId, err)
 	}
-	return nil
 }
 
 func socketCleanup(socketId int, socketConnection *SocketConnection) {
 	err := socketConnection.connection.Close()
 	if err != nil {
-		logger.Log.Errorf("error closing socket connection for socket id %d: %v", socketId, err)
+		logger.Log.Errorf("Error closing socket connection for socket id %d: %v", socketId, err)
 	}
 
 	websocketIdsLock.Lock()
@@ -154,3 +183,29 @@ func socketCleanup(socketId int, socketConnection *SocketConnection) {
 
 	socketConnection.eventHandlers.WebSocketDisconnect(socketId, socketConnection.isTapper)
 }
+
+var db = debounce.NewDebouncer(time.Second*5, func() {
+	logger.Log.Error("Successfully sent to socket")
+})
+
+func SendToSocket(socketId int, message []byte) error {
+	socketObj := connectedWebsockets[socketId]
+	if socketObj == nil {
+		return errors.New("Socket is disconnected")
+	}
+
+	var sent = false
+	time.AfterFunc(time.Second*5, func() {
+		if !sent {
+			logger.Log.Error("Socket timed out")
+			socketCleanup(socketId, socketObj)
+		}
+	})
+
+	socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
+	err := socketObj.connection.WriteMessage(1, message)
+	socketObj.lock.Unlock()
+
+	sent = true
+	return err
+}

agent/pkg/api/socket_server_handlers.go

@@ -1,29 +1,21 @@
 package api
 
 import (
-	"context"
 	"encoding/json"
+	"fmt"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/providers/tappers"
+	"mizuserver/pkg/up9"
 	"sync"
 
-	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/agent/pkg/dependency"
-	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
-	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
-	"github.com/up9inc/mizu/agent/pkg/up9"
-
 	tapApi "github.com/up9inc/mizu/tap/api"
 
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 )
 
-type BrowserClient struct {
-	dataStreamCancelFunc context.CancelFunc
-}
-
-var browserClients = make(map[int]*BrowserClient, 0)
-var tapperClientSocketUUIDs = make([]int, 0)
+var browserClientSocketUUIDs = make([]int, 0)
 var socketListLock = sync.Mutex{}
 
 type RoutesEventHandlers struct {
@@ -35,25 +27,15 @@ func init() {
 	go up9.UpdateAnalyzeStatus(BroadcastToBrowserClients)
 }
 
-func (h *RoutesEventHandlers) WebSocketConnect(_ *gin.Context, socketId int, isTapper bool) {
+func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
 		tappers.Connected()
-
-		socketListLock.Lock()
-		tapperClientSocketUUIDs = append(tapperClientSocketUUIDs, socketId)
-		socketListLock.Unlock()
-
-		nodeToTappedPodMap := tappedPods.GetNodeToTappedPodMap()
-		SendTappedPods(socketId, nodeToTappedPodMap)
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
-
 		socketListLock.Lock()
-		browserClients[socketId] = &BrowserClient{}
+		browserClientSocketUUIDs = append(browserClientSocketUUIDs, socketId)
 		socketListLock.Unlock()
-
-		BroadcastTappedPodsStatus()
 	}
 }
 
@@ -61,68 +43,26 @@ func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper disconnected, socket ID:  %d", socketId)
 		tappers.Disconnected()
-
-		socketListLock.Lock()
-		removeSocketUUIDFromTapperSlice(socketId)
-		socketListLock.Unlock()
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket disconnected, socket ID:  %d", socketId)
 		socketListLock.Lock()
-		if browserClients[socketId] != nil && browserClients[socketId].dataStreamCancelFunc != nil {
-			browserClients[socketId].dataStreamCancelFunc()
-		}
-		delete(browserClients, socketId)
+		removeSocketUUIDFromBrowserSlice(socketId)
 		socketListLock.Unlock()
 	}
 }
 
 func BroadcastToBrowserClients(message []byte) {
-	for socketId := range browserClients {
+	for _, socketId := range browserClientSocketUUIDs {
 		go func(socketId int) {
-			if err := SendToSocket(socketId, message); err != nil {
-				logger.Log.Error(err)
-			}
-		}(socketId)
-	}
-}
-
-func BroadcastToTapperClients(message []byte) {
-	for _, socketId := range tapperClientSocketUUIDs {
-		go func(socketId int) {
-			if err := SendToSocket(socketId, message); err != nil {
-				logger.Log.Error(err)
-			}
-		}(socketId)
-	}
-}
-
-func (h *RoutesEventHandlers) WebSocketMessage(socketId int, isTapper bool, message []byte) {
-	if isTapper {
-		HandleTapperIncomingMessage(message, h.SocketOutChannel, BroadcastToBrowserClients)
-	} else {
-		// we initiate the basenine stream after the first websocket message we receive (it contains the entry query), we then store a cancelfunc to later cancel this stream
-		if browserClients[socketId] != nil && browserClients[socketId].dataStreamCancelFunc == nil {
-			var params WebSocketParams
-			if err := json.Unmarshal(message, &params); err != nil {
-				logger.Log.Errorf("Error: %v", socketId, err)
-				return
-			}
-
-			entriesStreamer := dependency.GetInstance(dependency.EntriesSocketStreamer).(EntryStreamer)
-			ctx, cancelFunc := context.WithCancel(context.Background())
-			err := entriesStreamer.Get(ctx, socketId, &params)
-
+			err := SendToSocket(socketId, message)
 			if err != nil {
-				logger.Log.Errorf("error initializing basenine stream for browser socket %d %+v", socketId, err)
-				cancelFunc()
-			} else {
-				browserClients[socketId].dataStreamCancelFunc = cancelFunc
+				logger.Log.Errorf("error sending message to socket ID %d: %v", socketId, err)
 			}
-		}
+		}(socketId)
 	}
 }
 
-func HandleTapperIncomingMessage(message []byte, socketOutChannel chan<- *tapApi.OutputChannelItem, broadcastMessageFunc func([]byte)) {
+func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 	var socketMessageBase shared.WebSocketMessageMetadata
 	err := json.Unmarshal(message, &socketMessageBase)
 	if err != nil {
@@ -136,7 +76,7 @@ func HandleTapperIncomingMessage(message []byte, socketOutChannel chan<- *tapApi
 				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
 			} else {
 				// NOTE: This is where the message comes back from the intermediate WebSocket to code.
-				socketOutChannel <- tappedEntryMessage.Data
+				h.SocketOutChannel <- tappedEntryMessage.Data
 			}
 		case shared.WebSocketMessageTypeUpdateStatus:
 			var statusMessage shared.WebSocketStatusMessage
@@ -144,7 +84,15 @@ func HandleTapperIncomingMessage(message []byte, socketOutChannel chan<- *tapApi
 			if err != nil {
 				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
 			} else {
-				broadcastMessageFunc(message)
+				BroadcastToBrowserClients(message)
+			}
+		case shared.WebsocketMessageTypeOutboundLink:
+			var outboundLinkMessage models.WebsocketOutboundLinkMessage
+			err := json.Unmarshal(message, &outboundLinkMessage)
+			if err != nil {
+				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
+			} else {
+				handleTLSLink(outboundLinkMessage)
 			}
 		default:
 			logger.Log.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
@@ -152,12 +100,35 @@ func HandleTapperIncomingMessage(message []byte, socketOutChannel chan<- *tapApi
 	}
 }
 
-func removeSocketUUIDFromTapperSlice(uuidToRemove int) {
-	newUUIDSlice := make([]int, 0, len(tapperClientSocketUUIDs))
-	for _, uuid := range tapperClientSocketUUIDs {
+func handleTLSLink(outboundLinkMessage models.WebsocketOutboundLinkMessage) {
+	resolvedName := k8sResolver.Resolve(outboundLinkMessage.Data.DstIP)
+	if resolvedName != "" {
+		outboundLinkMessage.Data.DstIP = resolvedName
+	} else if outboundLinkMessage.Data.SuggestedResolvedName != "" {
+		outboundLinkMessage.Data.DstIP = outboundLinkMessage.Data.SuggestedResolvedName
+	}
+	cacheKey := fmt.Sprintf("%s -> %s:%d", outboundLinkMessage.Data.Src, outboundLinkMessage.Data.DstIP, outboundLinkMessage.Data.DstPort)
+	_, isInCache := providers.RecentTLSLinks.Get(cacheKey)
+	if isInCache {
+		return
+	} else {
+		providers.RecentTLSLinks.SetDefault(cacheKey, outboundLinkMessage.Data)
+	}
+	marshaledMessage, err := json.Marshal(outboundLinkMessage)
+	if err != nil {
+		logger.Log.Errorf("Error marshaling outbound link message for broadcasting: %v", err)
+	} else {
+		logger.Log.Errorf("Broadcasting outboundlink message %s", string(marshaledMessage))
+		BroadcastToBrowserClients(marshaledMessage)
+	}
+}
+
+func removeSocketUUIDFromBrowserSlice(uuidToRemove int) {
+	newUUIDSlice := make([]int, 0, len(browserClientSocketUUIDs))
+	for _, uuid := range browserClientSocketUUIDs {
 		if uuid != uuidToRemove {
 			newUUIDSlice = append(newUUIDSlice, uuid)
 		}
 	}
-	tapperClientSocketUUIDs = newUUIDSlice
+	browserClientSocketUUIDs = newUUIDSlice
 }

agent/pkg/api/utils.go

@@ -1,40 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-
-	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-func BroadcastTappedPodsStatus() {
-	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
-
-	message := shared.CreateWebSocketStatusMessage(tappedPodsStatus)
-	if jsonBytes, err := json.Marshal(message); err != nil {
-		logger.Log.Errorf("Could not Marshal message %v", err)
-	} else {
-		BroadcastToBrowserClients(jsonBytes)
-	}
-}
-
-func SendTappedPods(socketId int, nodeToTappedPodMap shared.NodeToPodsMap) {
-	message := shared.CreateWebSocketTappedPodsMessage(nodeToTappedPodMap)
-	if jsonBytes, err := json.Marshal(message); err != nil {
-		logger.Log.Errorf("Could not Marshal message %v", err)
-	} else {
-		if err := SendToSocket(socketId, jsonBytes); err != nil {
-			logger.Log.Error(err)
-		}
-	}
-}
-
-func BroadcastTappedPodsToTappers(nodeToTappedPodMap shared.NodeToPodsMap) {
-	message := shared.CreateWebSocketTappedPodsMessage(nodeToTappedPodMap)
-	if jsonBytes, err := json.Marshal(message); err != nil {
-		logger.Log.Errorf("Could not Marshal message %v", err)
-	} else {
-		BroadcastToTapperClients(jsonBytes)
-	}
-}

agent/pkg/app/main.go

@@ -1,115 +0,0 @@
-package app
-
-import (
-	"fmt"
-	"sort"
-	"time"
-
-	"github.com/antelman107/net-wait-go/wait"
-	"github.com/op/go-logging"
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/agent/pkg/api"
-	"github.com/up9inc/mizu/agent/pkg/utils"
-	"github.com/up9inc/mizu/shared/logger"
-	tapApi "github.com/up9inc/mizu/tap/api"
-	amqpExt "github.com/up9inc/mizu/tap/extensions/amqp"
-	httpExt "github.com/up9inc/mizu/tap/extensions/http"
-	kafkaExt "github.com/up9inc/mizu/tap/extensions/kafka"
-	redisExt "github.com/up9inc/mizu/tap/extensions/redis"
-)
-
-var (
-	Extensions    []*tapApi.Extension          // global
-	ExtensionsMap map[string]*tapApi.Extension // global
-)
-
-func LoadExtensions() {
-	Extensions = make([]*tapApi.Extension, 4)
-	ExtensionsMap = make(map[string]*tapApi.Extension)
-
-	extensionAmqp := &tapApi.Extension{}
-	dissectorAmqp := amqpExt.NewDissector()
-	dissectorAmqp.Register(extensionAmqp)
-	extensionAmqp.Dissector = dissectorAmqp
-	Extensions[0] = extensionAmqp
-	ExtensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
-
-	extensionHttp := &tapApi.Extension{}
-	dissectorHttp := httpExt.NewDissector()
-	dissectorHttp.Register(extensionHttp)
-	extensionHttp.Dissector = dissectorHttp
-	Extensions[1] = extensionHttp
-	ExtensionsMap[extensionHttp.Protocol.Name] = extensionHttp
-
-	extensionKafka := &tapApi.Extension{}
-	dissectorKafka := kafkaExt.NewDissector()
-	dissectorKafka.Register(extensionKafka)
-	extensionKafka.Dissector = dissectorKafka
-	Extensions[2] = extensionKafka
-	ExtensionsMap[extensionKafka.Protocol.Name] = extensionKafka
-
-	extensionRedis := &tapApi.Extension{}
-	dissectorRedis := redisExt.NewDissector()
-	dissectorRedis.Register(extensionRedis)
-	extensionRedis.Dissector = dissectorRedis
-	Extensions[3] = extensionRedis
-	ExtensionsMap[extensionRedis.Protocol.Name] = extensionRedis
-
-	sort.Slice(Extensions, func(i, j int) bool {
-		return Extensions[i].Protocol.Priority < Extensions[j].Protocol.Priority
-	})
-
-	api.InitExtensionsMap(ExtensionsMap)
-}
-
-func ConfigureBasenineServer(host string, port string, dbSize int64, logLevel logging.Level, insertionFilter string) {
-	if !wait.New(
-		wait.WithProto("tcp"),
-		wait.WithWait(200*time.Millisecond),
-		wait.WithBreak(50*time.Millisecond),
-		wait.WithDeadline(20*time.Second),
-		wait.WithDebug(logLevel == logging.DEBUG),
-	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
-		logger.Log.Panicf("Basenine is not available!")
-	}
-
-	if err := basenine.Limit(host, port, dbSize); err != nil {
-		logger.Log.Panicf("Error while limiting database size: %v", err)
-	}
-
-	// Define the macros
-	for _, extension := range Extensions {
-		macros := extension.Dissector.Macros()
-		for macro, expanded := range macros {
-			if err := basenine.Macro(host, port, macro, expanded); err != nil {
-				logger.Log.Panicf("Error while adding a macro: %v", err)
-			}
-		}
-	}
-
-	// Set the insertion filter that comes from the config
-	if err := basenine.InsertionFilter(host, port, insertionFilter); err != nil {
-		logger.Log.Errorf("Error while setting the insertion filter: %v", err)
-	}
-
-	utils.StartTime = time.Now().UnixNano() / int64(time.Millisecond)
-}
-
-func GetEntryInputChannel() chan *tapApi.OutputChannelItem {
-	outputItemsChannel := make(chan *tapApi.OutputChannelItem)
-	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-	go FilterItems(outputItemsChannel, filteredOutputItemsChannel)
-	go api.StartReadingEntries(filteredOutputItemsChannel, nil, ExtensionsMap)
-
-	return outputItemsChannel
-}
-
-func FilterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem) {
-	for message := range inChannel {
-		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
-			continue
-		}
-
-		outChannel <- message
-	}
-}

agent/pkg/controllers/config_controller.go

@@ -0,0 +1,154 @@
+package controllers
+
+import (
+	"context"
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
+	v1 "k8s.io/api/core/v1"
+	"mizuserver/pkg/config"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/providers/tapConfig"
+	"mizuserver/pkg/providers/tappedPods"
+	"mizuserver/pkg/providers/tappers"
+	"net/http"
+	"regexp"
+	"time"
+)
+
+var cancelTapperSyncer context.CancelFunc
+
+func PostTapConfig(c *gin.Context) {
+	requestTapConfig := &models.TapConfig{}
+
+	if err := c.Bind(requestTapConfig); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+
+	if cancelTapperSyncer != nil {
+		cancelTapperSyncer()
+
+		tappedPods.Set([]*shared.PodInfo{})
+		tappers.ResetStatus()
+
+		broadcastTappedPodsStatus()
+	}
+
+	var tappedNamespaces []string
+	for namespace, tapped := range requestTapConfig.TappedNamespaces {
+		if tapped {
+			tappedNamespaces = append(tappedNamespaces, namespace)
+		}
+	}
+
+	podRegex, _ := regexp.Compile(".*")
+
+	kubernetesProvider, err := providers.GetKubernetesProvider()
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, err)
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	if _, err := startMizuTapperSyncer(ctx, kubernetesProvider, tappedNamespaces, *podRegex, []string{}, tapApi.TrafficFilteringOptions{}, false); err != nil {
+		c.JSON(http.StatusInternalServerError, err)
+		cancel()
+		return
+	}
+
+	cancelTapperSyncer = cancel
+	tapConfig.Save(requestTapConfig)
+
+	c.JSON(http.StatusOK, "OK")
+}
+
+func GetTapConfig(c *gin.Context) {
+	kubernetesProvider, err := providers.GetKubernetesProvider()
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, err)
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	namespaces, err := kubernetesProvider.ListAllNamespaces(ctx)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, err)
+		return
+	}
+
+	savedTapConfig := tapConfig.Get()
+
+	tappedNamespaces := make(map[string]bool)
+	for _, namespace := range namespaces {
+		if namespace.Name == config.Config.MizuResourcesNamespace {
+			continue
+		}
+
+		tappedNamespaces[namespace.Name] = savedTapConfig.TappedNamespaces[namespace.Name]
+	}
+
+	tapConfigToReturn := models.TapConfig{TappedNamespaces: tappedNamespaces}
+	c.JSON(http.StatusOK, tapConfigToReturn)
+}
+
+func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, serviceMesh bool) (*kubernetes.MizuTapperSyncer, error) {
+	tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
+		TargetNamespaces:         targetNamespaces,
+		PodFilterRegex:           podFilterRegex,
+		MizuResourcesNamespace:   config.Config.MizuResourcesNamespace,
+		AgentImage:               config.Config.AgentImage,
+		TapperResources:          config.Config.TapperResources,
+		ImagePullPolicy:          v1.PullPolicy(config.Config.PullPolicy),
+		LogLevel:                 config.Config.LogLevel,
+		IgnoredUserAgents:        ignoredUserAgents,
+		MizuApiFilteringOptions:  mizuApiFilteringOptions,
+		MizuServiceAccountExists: true, //assume service account exists since install mode will not function without it anyway
+		ServiceMesh:              serviceMesh,
+	}, time.Now())
+
+	if err != nil {
+		return nil, err
+	}
+
+	// handle tapperSyncer events (pod changes and errors)
+	go func() {
+		for {
+			select {
+			case syncerErr, ok := <-tapperSyncer.ErrorOut:
+				if !ok {
+					logger.Log.Debug("mizuTapperSyncer err channel closed, ending listener loop")
+					return
+				}
+				logger.Log.Fatalf("fatal tap syncer error: %v", syncerErr)
+			case _, ok := <-tapperSyncer.TapPodChangesOut:
+				if !ok {
+					logger.Log.Debug("mizuTapperSyncer pod changes channel closed, ending listener loop")
+					return
+				}
+
+				tappedPods.Set(kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods))
+				broadcastTappedPodsStatus()
+			case tapperStatus, ok := <-tapperSyncer.TapperStatusChangedOut:
+				if !ok {
+					logger.Log.Debug("mizuTapperSyncer tapper status changed channel closed, ending listener loop")
+					return
+				}
+
+				tappers.SetStatus(&tapperStatus)
+				broadcastTappedPodsStatus()
+			case <-ctx.Done():
+				logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")
+				return
+			}
+		}
+	}()
+
+	return tapperSyncer, nil
+}

agent/pkg/controllers/db_controller.go

@@ -1,28 +0,0 @@
-package controllers
-
-import (
-	"net/http"
-
-	"github.com/gin-gonic/gin"
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/agent/pkg/app"
-	"github.com/up9inc/mizu/agent/pkg/config"
-	"github.com/up9inc/mizu/shared"
-)
-
-func Flush(c *gin.Context) {
-	if err := basenine.Flush(shared.BasenineHost, shared.BaseninePort); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-	} else {
-		c.JSON(http.StatusOK, "Flushed.")
-	}
-}
-
-func Reset(c *gin.Context) {
-	if err := basenine.Reset(shared.BasenineHost, shared.BaseninePort); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-	} else {
-		app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort, config.Config.MaxDBSizeBytes, config.Config.LogLevel, config.Config.InsertionFilter)
-		c.JSON(http.StatusOK, "Resetted.")
-	}
-}

agent/pkg/controllers/entries_controller.go

@@ -1,22 +1,32 @@
 package controllers
 
 import (
+	"encoding/json"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/utils"
+	"mizuserver/pkg/validation"
 	"net/http"
-
-	"github.com/up9inc/mizu/agent/pkg/dependency"
-	"github.com/up9inc/mizu/agent/pkg/entries"
-	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/agent/pkg/validation"
+	"strconv"
+	"time"
 
 	"github.com/gin-gonic/gin"
 
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
-func HandleEntriesError(c *gin.Context, err error) bool {
+var extensionsMap map[string]*tapApi.Extension // global
+
+func InitExtensionsMap(ref map[string]*tapApi.Extension) {
+	extensionsMap = ref
+}
+
+func Error(c *gin.Context, err error) bool {
 	if err != nil {
 		logger.Log.Errorf("Error getting entry: %v", err)
-		_ = c.Error(err)
+		c.Error(err)
 		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
 			"error":     true,
 			"type":      "error",
@@ -43,18 +53,44 @@ func GetEntries(c *gin.Context) {
 		entriesRequest.TimeoutMs = 3000
 	}
 
-	entriesProvider := dependency.GetInstance(dependency.EntriesProvider).(entries.EntriesProvider)
-	entries, metadata, err := entriesProvider.GetEntries(entriesRequest)
-	if !HandleEntriesError(c, err) {
-		baseEntries := make([]interface{}, 0)
-		for _, entry := range entries {
-			baseEntries = append(baseEntries, entry.Base)
-		}
-		c.JSON(http.StatusOK, models.EntriesResponse{
-			Data: baseEntries,
-			Meta: metadata,
-		})
+	data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
+		entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
+		entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, validationError)
 	}
+
+	response := &models.EntriesResponse{}
+	var dataSlice []interface{}
+
+	for _, row := range data {
+		var entry *tapApi.Entry
+		err = json.Unmarshal(row, &entry)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{
+				"error":     true,
+				"type":      "error",
+				"autoClose": "5000",
+				"msg":       string(row),
+			})
+			return // exit
+		}
+
+		base := tapApi.Summarize(entry)
+
+		dataSlice = append(dataSlice, base)
+	}
+
+	var metadata *basenine.Metadata
+	err = json.Unmarshal(meta, &metadata)
+	if err != nil {
+		logger.Log.Debugf("Error recieving metadata: %v", err.Error())
+	}
+
+	response.Data = dataSlice
+	response.Meta = metadata
+
+	c.JSON(http.StatusOK, response)
 }
 
 func GetEntry(c *gin.Context) {
@@ -68,12 +104,42 @@ func GetEntry(c *gin.Context) {
 		c.JSON(http.StatusBadRequest, validationError)
 	}
 
-	id := c.Param("id")
-
-	entriesProvider := dependency.GetInstance(dependency.EntriesProvider).(entries.EntriesProvider)
-	entry, err := entriesProvider.GetEntry(singleEntryRequest, id)
-
-	if !HandleEntriesError(c, err) {
-		c.JSON(http.StatusOK, entry)
+	id, _ := strconv.Atoi(c.Param("id"))
+	var entry *tapApi.Entry
+	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id, singleEntryRequest.Query)
+	if Error(c, err) {
+		return // exit
 	}
+	err = json.Unmarshal(bytes, &entry)
+	if err != nil {
+		c.JSON(http.StatusNotFound, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       string(bytes),
+		})
+		return // exit
+	}
+
+	extension := extensionsMap[entry.Protocol.Name]
+	representation, bodySize, _ := extension.Dissector.Represent(entry.Request, entry.Response)
+
+	var rules []map[string]interface{}
+	var isRulesEnabled bool
+	if entry.Protocol.Name == "http" {
+		harEntry, _ := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
+		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
+		isRulesEnabled = _isRulesEnabled
+		inrec, _ := json.Marshal(rulesMatched)
+		json.Unmarshal(inrec, &rules)
+	}
+
+	c.JSON(http.StatusOK, tapApi.EntryWrapper{
+		Protocol:       entry.Protocol,
+		Representation: string(representation),
+		BodySize:       bodySize,
+		Data:           entry,
+		Rules:          rules,
+		IsRulesEnabled: isRulesEnabled,
+	})
 }

agent/pkg/controllers/install_controller.go

@@ -0,0 +1,18 @@
+package controllers
+
+import (
+	"mizuserver/pkg/providers"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/shared/logger"
+)
+
+func IsSetupNecessary(c *gin.Context) {
+	if IsInstallNeeded, err := providers.IsInstallNeeded(); err != nil {
+		logger.Log.Errorf("unknown internal while checking if install is needed %s", err)
+		c.AbortWithStatusJSON(500, gin.H{"error": "internal error occured while checking if install is needed"})
+	} else {
+		c.JSON(http.StatusOK, IsInstallNeeded)
+	}
+}

agent/pkg/controllers/metadata_controller.go

@@ -1,14 +1,13 @@
 package controllers
 
 import (
-	"net/http"
-
 	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/agent/pkg/version"
 	"github.com/up9inc/mizu/shared"
+	"mizuserver/pkg/version"
+	"net/http"
 )
 
 func GetVersion(c *gin.Context) {
-	resp := shared.VersionResponse{Ver: version.Ver}
+	resp := shared.VersionResponse{SemVer: version.SemVer}
 	c.JSON(http.StatusOK, resp)
 }

agent/pkg/controllers/oas_controller.go

@@ -1,19 +1,16 @@
 package controllers
 
 import (
-	"net/http"
-
 	"github.com/chanced/openapi"
 	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/agent/pkg/dependency"
-	"github.com/up9inc/mizu/agent/pkg/oas"
 	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/oas"
+	"net/http"
 )
 
 func GetOASServers(c *gin.Context) {
 	m := make([]string, 0)
-	oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
-	oasGenerator.GetServiceSpecs().Range(func(key, value interface{}) bool {
+	oas.GetOasGeneratorInstance().ServiceSpecs.Range(func(key, value interface{}) bool {
 		m = append(m, key.(string))
 		return true
 	})
@@ -22,8 +19,7 @@ func GetOASServers(c *gin.Context) {
 }
 
 func GetOASSpec(c *gin.Context) {
-	oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
-	res, ok := oasGenerator.GetServiceSpecs().Load(c.Param("id"))
+	res, ok := oas.GetOasGeneratorInstance().ServiceSpecs.Load(c.Param("id"))
 	if !ok {
 		c.JSON(http.StatusNotFound, gin.H{
 			"error":     true,
@@ -51,9 +47,7 @@ func GetOASSpec(c *gin.Context) {
 
 func GetOASAllSpecs(c *gin.Context) {
 	res := map[string]*openapi.OpenAPI{}
-
-	oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
-	oasGenerator.GetServiceSpecs().Range(func(key, value interface{}) bool {
+	oas.GetOasGeneratorInstance().ServiceSpecs.Range(func(key, value interface{}) bool {
 		svc := key.(string)
 		gen := value.(*oas.SpecGen)
 		spec, err := gen.GetSpec()

agent/pkg/controllers/oas_controller_test.go

@@ -1,69 +1,43 @@
 package controllers
 
 import (
-	"bytes"
-	basenine "github.com/up9inc/basenine/client/go"
-	"net"
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/oas"
 	"net/http/httptest"
 	"testing"
-	"time"
-
-	"github.com/up9inc/mizu/agent/pkg/dependency"
-	"github.com/up9inc/mizu/agent/pkg/oas"
-
-	"github.com/gin-gonic/gin"
 )
 
 func TestGetOASServers(t *testing.T) {
-	recorder, c := getRecorderAndContext()
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetOasGeneratorInstance().Start()
+	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
 
 	GetOASServers(c)
 	t.Logf("Written body: %s", recorder.Body.String())
+	return
 }
 
 func TestGetOASAllSpecs(t *testing.T) {
-	recorder, c := getRecorderAndContext()
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetOasGeneratorInstance().Start()
+	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
 
 	GetOASAllSpecs(c)
 	t.Logf("Written body: %s", recorder.Body.String())
+	return
 }
 
 func TestGetOASSpec(t *testing.T) {
-	recorder, c := getRecorderAndContext()
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetOasGeneratorInstance().Start()
+	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
 
 	c.Params = []gin.Param{{Key: "id", Value: "some"}}
 
 	GetOASSpec(c)
 	t.Logf("Written body: %s", recorder.Body.String())
-}
-
-type fakeConn struct {
-	sendBuffer    *bytes.Buffer
-	receiveBuffer *bytes.Buffer
-}
-
-func (f fakeConn) Read(p []byte) (int, error)       { return f.sendBuffer.Read(p) }
-func (f fakeConn) Write(p []byte) (int, error)      { return f.receiveBuffer.Write(p) }
-func (fakeConn) Close() error                       { return nil }
-func (fakeConn) LocalAddr() net.Addr                { return nil }
-func (fakeConn) RemoteAddr() net.Addr               { return nil }
-func (fakeConn) SetDeadline(t time.Time) error      { return nil }
-func (fakeConn) SetReadDeadline(t time.Time) error  { return nil }
-func (fakeConn) SetWriteDeadline(t time.Time) error { return nil }
-
-func getRecorderAndContext() (*httptest.ResponseRecorder, *gin.Context) {
-	dummyConn := new(basenine.Connection)
-	dummyConn.Conn = fakeConn{
-		sendBuffer:    bytes.NewBufferString("\n"),
-		receiveBuffer: bytes.NewBufferString("\n"),
-	}
-	dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} {
-		return oas.GetDefaultOasGeneratorInstance()
-	})
-
-	recorder := httptest.NewRecorder()
-	c, _ := gin.CreateTestContext(recorder)
-	oas.GetDefaultOasGeneratorInstance().Start(dummyConn)
-	oas.GetDefaultOasGeneratorInstance().GetServiceSpecs().Store("some", oas.NewGen("some"))
-	return recorder, c
+	return
 }

agent/pkg/controllers/service_map_controller.go

@@ -1,11 +1,9 @@
 package controllers
 
 import (
+	"mizuserver/pkg/servicemap"
 	"net/http"
 
-	"github.com/up9inc/mizu/agent/pkg/dependency"
-	"github.com/up9inc/mizu/agent/pkg/servicemap"
-
 	"github.com/gin-gonic/gin"
 )
 
@@ -14,9 +12,8 @@ type ServiceMapController struct {
 }
 
 func NewServiceMapController() *ServiceMapController {
-	serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMap)
 	return &ServiceMapController{
-		service: serviceMapGenerator,
+		service: servicemap.GetInstance(),
 	}
 }
 

agent/pkg/controllers/service_map_controller_test.go

@@ -7,11 +7,11 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/up9inc/mizu/agent/pkg/dependency"
-	"github.com/up9inc/mizu/agent/pkg/servicemap"
+	"mizuserver/pkg/servicemap"
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	"github.com/up9inc/mizu/shared"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
@@ -58,11 +58,11 @@ type ServiceMapControllerSuite struct {
 }
 
 func (s *ServiceMapControllerSuite) SetupTest() {
-	dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
-
 	s.c = NewServiceMapController()
-	s.c.service.Enable()
-	s.c.service.(servicemap.ServiceMapSink).NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
+	s.c.service.SetConfig(&shared.MizuAgentConfig{
+		ServiceMap: true,
+	})
+	s.c.service.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
 
 	s.w = httptest.NewRecorder()
 	s.g, _ = gin.CreateTestContext(s.w)
@@ -101,18 +101,16 @@ func (s *ServiceMapControllerSuite) TestGet() {
 
 	// response nodes
 	aNode := servicemap.ServiceMapNode{
-		Id:       1,
-		Name:     TCPEntryA.Name,
-		Entry:    TCPEntryA,
-		Resolved: true,
-		Count:    1,
+		Id:    1,
+		Name:  TCPEntryA.IP,
+		Entry: TCPEntryA,
+		Count: 1,
 	}
 	bNode := servicemap.ServiceMapNode{
-		Id:       2,
-		Name:     TCPEntryB.Name,
-		Entry:    TCPEntryB,
-		Resolved: true,
-		Count:    1,
+		Id:    2,
+		Name:  TCPEntryB.IP,
+		Entry: TCPEntryB,
+		Count: 1,
 	}
 	assert.Contains(response.Nodes, aNode)
 	assert.Contains(response.Nodes, bNode)

agent/pkg/controllers/status_controller.go

@@ -1,20 +1,18 @@
 package controllers
 
 import (
-	"net/http"
-	core "k8s.io/api/core/v1"
-
+	"encoding/json"
 	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/agent/pkg/api"
-	"github.com/up9inc/mizu/agent/pkg/holder"
-	"github.com/up9inc/mizu/agent/pkg/providers"
-	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
-	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
-	"github.com/up9inc/mizu/agent/pkg/up9"
-	"github.com/up9inc/mizu/agent/pkg/validation"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/kubernetes"
 	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/api"
+	"mizuserver/pkg/holder"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/providers/tappedPods"
+	"mizuserver/pkg/providers/tappers"
+	"mizuserver/pkg/up9"
+	"mizuserver/pkg/validation"
+	"net/http"
 )
 
 func HealthCheck(c *gin.Context) {
@@ -32,21 +30,26 @@ func HealthCheck(c *gin.Context) {
 }
 
 func PostTappedPods(c *gin.Context) {
-	var requestTappedPods []core.Pod
+	var requestTappedPods []*shared.PodInfo
 	if err := c.Bind(&requestTappedPods); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
 
-	podInfos := kubernetes.GetPodInfosForPods(requestTappedPods)
-
 	logger.Log.Infof("[Status] POST request: %d tapped pods", len(requestTappedPods))
-	tappedPods.Set(podInfos)
-	api.BroadcastTappedPodsStatus()
+	tappedPods.Set(requestTappedPods)
+	broadcastTappedPodsStatus()
+}
 
-	nodeToTappedPodMap := kubernetes.GetNodeHostToTappedPodsMap(requestTappedPods)
-	tappedPods.SetNodeToTappedPodMap(nodeToTappedPodMap)
-	api.BroadcastTappedPodsToTappers(nodeToTappedPodMap)
+func broadcastTappedPodsStatus() {
+	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
+
+	message := shared.CreateWebSocketStatusMessage(tappedPodsStatus)
+	if jsonBytes, err := json.Marshal(message); err != nil {
+		logger.Log.Errorf("Could not Marshal message %v", err)
+	} else {
+		api.BroadcastToBrowserClients(jsonBytes)
+	}
 }
 
 func PostTapperStatus(c *gin.Context) {
@@ -63,7 +66,7 @@ func PostTapperStatus(c *gin.Context) {
 
 	logger.Log.Infof("[Status] POST request, tapper status: %v", tapperStatus)
 	tappers.SetStatus(tapperStatus)
-	api.BroadcastTappedPodsStatus()
+	broadcastTappedPodsStatus()
 }
 
 func GetConnectedTappersCount(c *gin.Context) {

agent/pkg/controllers/user_controller.go

@@ -0,0 +1,39 @@
+package controllers
+
+import (
+	"mizuserver/pkg/providers"
+
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/shared/logger"
+)
+
+func Login(c *gin.Context) {
+	if token, err := providers.PerformLogin(c.PostForm("username"), c.PostForm("password"), c.Request.Context()); err != nil {
+		c.AbortWithStatusJSON(401, gin.H{"error": "bad login"})
+	} else {
+		c.JSON(200, gin.H{"token": token})
+	}
+}
+
+func Logout(c *gin.Context) {
+	token := c.GetHeader("x-session-token")
+	if err := providers.Logout(token, c.Request.Context()); err != nil {
+		c.AbortWithStatusJSON(500, gin.H{"error": "error occured while logging out, the session might still be valid"})
+	} else {
+		c.JSON(200, "")
+	}
+}
+
+func Register(c *gin.Context) {
+	if token, _, err, formErrorMessages := providers.RegisterUser(c.PostForm("username"), c.PostForm("password"), c.Request.Context()); err != nil {
+		if formErrorMessages != nil {
+			logger.Log.Infof("user attempted to register but had form errors %v %v", formErrorMessages, err)
+			c.AbortWithStatusJSON(400, formErrorMessages)
+		} else {
+			logger.Log.Errorf("unknown internal error registering user %s", err)
+			c.AbortWithStatusJSON(500, gin.H{"error": "internal error occured while registering"})
+		}
+	} else {
+		c.JSON(200, gin.H{"token": token})
+	}
+}

agent/pkg/dependency/container.go

@@ -1,11 +0,0 @@
-package dependency
-
-var typeIntializerMap = make(map[DependencyContainerType]func() interface{}, 0)
-
-func RegisterGenerator(name DependencyContainerType, fn func() interface{}) {
-	typeIntializerMap[name] = fn
-}
-
-func GetInstance(name DependencyContainerType) interface{} {
-	return typeIntializerMap[name]()
-}

agent/pkg/dependency/type_names.go

@@ -1,11 +0,0 @@
-package dependency
-
-type DependencyContainerType string
-
-const (
-	ServiceMapGeneratorDependency = "ServiceMapGeneratorDependency"
-	OasGeneratorDependency        = "OasGeneratorDependency"
-	EntriesProvider               = "EntriesProvider"
-	EntriesSocketStreamer         = "EntriesSocketStreamer"
-	EntryStreamerSocketConnector  = "EntryStreamerSocketConnector"
-)

agent/pkg/elastic/esClient.go

@@ -1,116 +0,0 @@
-package elastic
-
-import (
-	"bytes"
-	"crypto/tls"
-	"encoding/json"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/elastic/go-elasticsearch/v7"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	"github.com/up9inc/mizu/tap/api"
-)
-
-type client struct {
-	es            *elasticsearch.Client
-	index         string
-	insertedCount int
-}
-
-var instance *client
-var once sync.Once
-
-func GetInstance() *client {
-	once.Do(func() {
-		instance = newClient()
-	})
-	return instance
-}
-
-func (client *client) Configure(config shared.ElasticConfig) {
-	if config.Url == "" || config.User == "" || config.Password == "" {
-		if client.es != nil {
-			client.es = nil
-		}
-		logger.Log.Infof("No elastic configuration was supplied, elastic exporter disabled")
-		return
-	}
-	transport := http.DefaultTransport
-	tlsClientConfig := &tls.Config{InsecureSkipVerify: true}
-	transport.(*http.Transport).TLSClientConfig = tlsClientConfig
-	cfg := elasticsearch.Config{
-		Addresses: []string{config.Url},
-		Username:  config.User,
-		Password:  config.Password,
-		Transport: transport,
-	}
-
-	es, err := elasticsearch.NewClient(cfg)
-	if err != nil {
-		logger.Log.Errorf("Failed to initialize elastic client %v", err)
-	}
-
-	// Have the client instance return a response
-	res, err := es.Info()
-	if err != nil {
-		logger.Log.Errorf("Elastic client.Info() ERROR: %v", err)
-	} else {
-		client.es = es
-		client.index = "mizu_traffic_http_" + time.Now().Format("2006_01_02_15_04")
-		client.insertedCount = 0
-		logger.Log.Infof("Elastic client configured, index: %s, cluster info: %v", client.index, res)
-	}
-	defer res.Body.Close()
-}
-
-func newClient() *client {
-	return &client{
-		es:    nil,
-		index: "",
-	}
-}
-
-type httpEntry struct {
-	Source      *api.TCP               `json:"src"`
-	Destination *api.TCP               `json:"dst"`
-	Outgoing    bool                   `json:"outgoing"`
-	CreatedAt   time.Time              `json:"createdAt"`
-	Request     map[string]interface{} `json:"request"`
-	Response    map[string]interface{} `json:"response"`
-	ElapsedTime int64                  `json:"elapsedTime"`
-}
-
-func (client *client) PushEntry(entry *api.Entry) {
-	if client.es == nil {
-		return
-	}
-
-	if entry.Protocol.Name != "http" {
-		return
-	}
-
-	entryToPush := httpEntry{
-		Source:      entry.Source,
-		Destination: entry.Destination,
-		Outgoing:    entry.Outgoing,
-		CreatedAt:   entry.StartTime,
-		Request:     entry.Request,
-		Response:    entry.Response,
-		ElapsedTime: entry.ElapsedTime,
-	}
-
-	entryJson, err := json.Marshal(entryToPush)
-	if err != nil {
-		logger.Log.Errorf("json.Marshal ERROR: %v", err)
-		return
-	}
-	var buffer bytes.Buffer
-	buffer.WriteString(string(entryJson))
-	res, _ := client.es.Index(client.index, &buffer)
-	if res.StatusCode == 201 {
-		client.insertedCount += 1
-	}
-}

agent/pkg/entries/entries_provider.go

@@ -1,98 +0,0 @@
-package entries
-
-import (
-	"encoding/json"
-	"time"
-
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/agent/pkg/app"
-	"github.com/up9inc/mizu/agent/pkg/har"
-	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	tapApi "github.com/up9inc/mizu/tap/api"
-)
-
-type EntriesProvider interface {
-	GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error)
-	GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId string) (*tapApi.EntryWrapper, error)
-}
-
-type BasenineEntriesProvider struct{}
-
-func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error) {
-	data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
-		entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
-		entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	var dataSlice []*tapApi.EntryWrapper
-
-	for _, row := range data {
-		var entry *tapApi.Entry
-		err = json.Unmarshal(row, &entry)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		extension := app.ExtensionsMap[entry.Protocol.Name]
-		base := extension.Dissector.Summarize(entry)
-
-		dataSlice = append(dataSlice, &tapApi.EntryWrapper{
-			Protocol: entry.Protocol,
-			Data:     entry,
-			Base:     base,
-		})
-	}
-
-	var metadata *basenine.Metadata
-	err = json.Unmarshal(meta, &metadata)
-	if err != nil {
-		logger.Log.Debugf("Error recieving metadata: %v", err.Error())
-	}
-
-	return dataSlice, metadata, nil
-}
-
-func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId string) (*tapApi.EntryWrapper, error) {
-	var entry *tapApi.Entry
-	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, entryId, singleEntryRequest.Query)
-	if err != nil {
-		return nil, err
-	}
-	err = json.Unmarshal(bytes, &entry)
-	if err != nil {
-		return nil, err
-	}
-
-	extension := app.ExtensionsMap[entry.Protocol.Name]
-	base := extension.Dissector.Summarize(entry)
-	var representation []byte
-	representation, err = extension.Dissector.Represent(entry.Request, entry.Response)
-	if err != nil {
-		return nil, err
-	}
-
-	var rules []map[string]interface{}
-	var isRulesEnabled bool
-	if entry.Protocol.Name == "http" {
-		harEntry, _ := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
-		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
-		isRulesEnabled = _isRulesEnabled
-		inrec, _ := json.Marshal(rulesMatched)
-		if err := json.Unmarshal(inrec, &rules); err != nil {
-			logger.Log.Error(err)
-		}
-	}
-
-	return &tapApi.EntryWrapper{
-		Protocol:       entry.Protocol,
-		Representation: string(representation),
-		Data:           entry,
-		Base:           base,
-		Rules:          rules,
-		IsRulesEnabled: isRulesEnabled,
-	}, nil
-}

agent/pkg/har/types.go

@@ -1,375 +0,0 @@
-package har
-
-import (
-	"encoding/base64"
-	"github.com/up9inc/mizu/shared/logger"
-	"time"
-	"unicode/utf8"
-)
-
-/*
-HTTP Archive (HAR) format
-https://w3c.github.io/web-performance/specs/HAR/Overview.html
-*/
-
-// HAR is a container type for deserialization
-type HAR struct {
-	Log Log `json:"log"`
-}
-
-// Log represents the root of the exported data. This object MUST be present and its name MUST be "log".
-type Log struct {
-	// The object contains the following name/value pairs:
-
-	// Required. Version number of the format.
-	Version string `json:"version"`
-	// Required. An object of type creator that contains the name and version
-	// information of the log creator application.
-	Creator Creator `json:"creator"`
-	// Optional. An object of type browser that contains the name and version
-	// information of the user agent.
-	Browser Browser `json:"browser"`
-	// Optional. An array of objects of type page, each representing one exported
-	// (tracked) page. Leave out this field if the application does not support
-	// grouping by pages.
-	Pages []Page `json:"pages,omitempty"`
-	// Required. An array of objects of type entry, each representing one
-	// exported (tracked) HTTP request.
-	Entries []Entry `json:"entries"`
-	// Optional. A comment provided by the user or the application. Sorting
-	// entries by startedDateTime (starting from the oldest) is preferred way how
-	// to export data since it can make importing faster. However the reader
-	// application should always make sure the array is sorted (if required for
-	// the import).
-	Comment string `json:"comment"`
-}
-
-// Creator contains information about the log creator application
-type Creator struct {
-	// Required. The name of the application that created the log.
-	Name string `json:"name"`
-	// Required. The version number of the application that created the log.
-	Version string `json:"version"`
-	// Optional. A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// Browser that created the log
-type Browser struct {
-	// Required. The name of the browser that created the log.
-	Name string `json:"name"`
-	// Required. The version number of the browser that created the log.
-	Version string `json:"version"`
-	// Optional. A comment provided by the user or the browser.
-	Comment string `json:"comment"`
-}
-
-// Page object for every exported web page and one <entry> object for every HTTP request.
-// In case when an HTTP trace tool isn't able to group requests by a page,
-// the <pages> object is empty and individual requests doesn't have a parent page.
-type Page struct {
-	/* There is one <page> object for every exported web page and one <entry>
-	object for every HTTP request. In case when an HTTP trace tool isn't able to
-	group requests by a page, the <pages> object is empty and individual
-	requests doesn't have a parent page.
-	*/
-
-	// Date and time stamp for the beginning of the page load
-	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00).
-	StartedDateTime string `json:"startedDateTime"`
-	// Unique identifier of a page within the . Entries use it to refer the parent page.
-	ID string `json:"id"`
-	// Page title.
-	Title string `json:"title"`
-	// Detailed timing info about page load.
-	PageTiming PageTiming `json:"pageTiming"`
-	// (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// PageTiming describes timings for various events (states) fired during the page load.
-// All times are specified in milliseconds. If a time info is not available appropriate field is set to -1.
-type PageTiming struct {
-	// Content of the page loaded. Number of milliseconds since page load started
-	// (page.startedDateTime). Use -1 if the timing does not apply to the current
-	// request.
-	// Depeding on the browser, onContentLoad property represents DOMContentLoad
-	// event or document.readyState == interactive.
-	OnContentLoad int `json:"onContentLoad"`
-	// Page is loaded (onLoad event fired). Number of milliseconds since page
-	// load started (page.startedDateTime). Use -1 if the timing does not apply
-	// to the current request.
-	OnLoad int `json:"onLoad"`
-	// (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment"`
-}
-
-// Entry is a unique, optional Reference to the parent page.
-// Leave out this field if the application does not support grouping by pages.
-type Entry struct {
-	Pageref string `json:"pageref,omitempty"`
-	// Date and time stamp of the request start
-	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD).
-	StartedDateTime string `json:"startedDateTime"`
-	// Total elapsed time of the request in milliseconds. This is the sum of all
-	// timings available in the timings object (i.e. not including -1 values) .
-	Time int `json:"time"`
-	// Detailed info about the request.
-	Request Request `json:"request"`
-	// Detailed info about the response.
-	Response Response `json:"response"`
-	// Info about cache usage.
-	Cache Cache `json:"cache"`
-	// Detailed timing info about request/response round trip.
-	PageTimings PageTimings `json:"pageTimings"`
-	// optional (new in 1.2) IP address of the server that was connected
-	// (result of DNS resolution).
-	ServerIPAddress string `json:"serverIPAddress,omitempty"`
-	// optional (new in 1.2) Unique ID of the parent TCP/IP connection, can be
-	// the client port number. Note that a port number doesn't have to be unique
-	// identifier in cases where the port is shared for more connections. If the
-	// port isn't available for the application, any other unique connection ID
-	// can be used instead (e.g. connection index). Leave out this field if the
-	// application doesn't support this info.
-	Connection string `json:"connection,omitempty"`
-	// (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// Request contains detailed info about performed request.
-type Request struct {
-	// Request method (GET, POST, ...).
-	Method string `json:"method"`
-	// Absolute URL of the request (fragments are not included).
-	URL string `json:"url"`
-	// Request HTTP Version.
-	HTTPVersion string `json:"httpVersion"`
-	// List of cookie objects.
-	Cookies []Cookie `json:"cookies"`
-	// List of header objects.
-	Headers []NVP `json:"headers"`
-	// List of query parameter objects.
-	QueryString []NVP `json:"queryString"`
-	// Posted data.
-	PostData PostData `json:"postData"`
-	// Total number of bytes from the start of the HTTP request message until
-	// (and including) the double CRLF before the body. Set to -1 if the info
-	// is not available.
-	HeaderSize int `json:"headerSize"`
-	// Size of the request body (POST data payload) in bytes. Set to -1 if the
-	// info is not available.
-	BodySize int `json:"bodySize"`
-	// (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment"`
-}
-
-// Response contains detailed info about the response.
-type Response struct {
-	// Response status.
-	Status int `json:"status"`
-	// Response status description.
-	StatusText string `json:"statusText"`
-	// Response HTTP Version.
-	HTTPVersion string `json:"httpVersion"`
-	// List of cookie objects.
-	Cookies []Cookie `json:"cookies"`
-	// List of header objects.
-	Headers []NVP `json:"headers"`
-	// Details about the response body.
-	Content Content `json:"content"`
-	// Redirection target URL from the Location response header.
-	RedirectURL string `json:"redirectURL"`
-	// Total number of bytes from the start of the HTTP response message until
-	// (and including) the double CRLF before the body. Set to -1 if the info is
-	// not available.
-	// The size of received response-headers is computed only from headers that
-	// are really received from the server. Additional headers appended by the
-	// browser are not included in this number, but they appear in the list of
-	// header objects.
-	HeadersSize int `json:"headersSize"`
-	// Size of the received response body in bytes. Set to zero in case of
-	// responses coming from the cache (304). Set to -1 if the info is not
-	// available.
-	BodySize int `json:"bodySize"`
-	// optional (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// Cookie contains list of all cookies (used in <request> and <response> objects).
-type Cookie struct {
-	// The name of the cookie.
-	Name string `json:"name"`
-	// The cookie value.
-	Value string `json:"value"`
-	// optional The path pertaining to the cookie.
-	Path string `json:"path,omitempty"`
-	// optional The host of the cookie.
-	Domain string `json:"domain,omitempty"`
-	// optional Cookie expiration time.
-	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.123+02:00).
-	Expires string `json:"expires,omitempty"`
-	// optional Set to true if the cookie is HTTP only, false otherwise.
-	HTTPOnly bool `json:"httpOnly,omitempty"`
-	// optional (new in 1.2) True if the cookie was transmitted over ssl, false
-	// otherwise.
-	Secure bool `json:"secure,omitempty"`
-	// optional (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// NVP is simply a name/value pair with a comment
-type NVP struct {
-	Name    string `json:"name"`
-	Value   string `json:"value"`
-	Comment string `json:"comment,omitempty"`
-}
-
-// PostData describes posted data, if any (embedded in <request> object).
-type PostData struct {
-	//  Mime type of posted data.
-	MimeType string `json:"mimeType"`
-	//  List of posted parameters (in case of URL encoded parameters).
-	Params []PostParam `json:"params"`
-	//  Plain text posted data
-	Text string `json:"text"`
-	// optional (new in 1.2) A comment provided by the user or the
-	// application.
-	Comment string `json:"comment,omitempty"`
-}
-
-func (d PostData) B64Decoded() (bool, []byte, string) {
-	// there is a weird gap in HAR spec 1.2, that does not define encoding for binary POST bodies
-	// we have own convention of putting `base64` into comment field to handle it similar to response `Content`
-	return b64Decoded(d.Comment, d.Text)
-}
-
-// PostParam is a list of posted parameters, if any (embedded in <postData> object).
-type PostParam struct {
-	// name of a posted parameter.
-	Name string `json:"name"`
-	// optional value of a posted parameter or content of a posted file.
-	Value string `json:"value,omitempty"`
-	// optional name of a posted file.
-	FileName string `json:"fileName,omitempty"`
-	// optional content type of a posted file.
-	ContentType string `json:"contentType,omitempty"`
-	// optional (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// Content describes details about response content (embedded in <response> object).
-type Content struct {
-	// Length of the returned content in bytes. Should be equal to
-	// response.bodySize if there is no compression and bigger when the content
-	// has been compressed.
-	Size int `json:"size"`
-	// optional Number of bytes saved. Leave out this field if the information
-	// is not available.
-	Compression int `json:"compression,omitempty"`
-	// MIME type of the response text (value of the Content-Type response
-	// header). The charset attribute of the MIME type is included (if
-	// available).
-	MimeType string `json:"mimeType"`
-	// optional Response body sent from the server or loaded from the browser
-	// cache. This field is populated with textual content only. The text field
-	// is either HTTP decoded text or a encoded (e.g. "base64") representation of
-	// the response body. Leave out this field if the information is not
-	// available.
-	Text string `json:"text,omitempty"`
-	// optional (new in 1.2) Encoding used for response text field e.g
-	// "base64". Leave out this field if the text field is HTTP decoded
-	// (decompressed & unchunked), than trans-coded from its original character
-	// set into UTF-8.
-	Encoding string `json:"encoding,omitempty"`
-	// optional (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-func (c Content) B64Decoded() (bool, []byte, string) {
-	return b64Decoded(c.Encoding, c.Text)
-}
-
-func b64Decoded(enc string, text string) (isBinary bool, asBytes []byte, asString string) {
-	if enc == "base64" {
-		decoded, err := base64.StdEncoding.DecodeString(text)
-		if err != nil {
-			logger.Log.Warningf("Failed to decode content as base64: %s", text)
-			return false, []byte(text), text
-		}
-		valid := utf8.Valid(decoded)
-		return !valid, decoded, string(decoded)
-	} else {
-		return false, nil, text
-	}
-}
-
-// Cache contains info about a request coming from browser cache.
-type Cache struct {
-	// optional State of a cache entry before the request. Leave out this field
-	// if the information is not available.
-	BeforeRequest CacheObject `json:"beforeRequest,omitempty"`
-	// optional State of a cache entry after the request. Leave out this field if
-	// the information is not available.
-	AfterRequest CacheObject `json:"afterRequest,omitempty"`
-	// optional (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// CacheObject is used by both beforeRequest and afterRequest
-type CacheObject struct {
-	// optional - Expiration time of the cache entry.
-	Expires string `json:"expires,omitempty"`
-	// The last time the cache entry was opened.
-	LastAccess string `json:"lastAccess"`
-	// Etag
-	ETag string `json:"eTag"`
-	// The number of times the cache entry has been opened.
-	HitCount int `json:"hitCount"`
-	// optional (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// PageTimings describes various phases within request-response round trip.
-// All times are specified in milliseconds.
-type PageTimings struct {
-	Blocked int `json:"blocked,omitempty"`
-	// optional - Time spent in a queue waiting for a network connection. Use -1
-	// if the timing does not apply to the current request.
-	DNS int `json:"dns,omitempty"`
-	// optional - DNS resolution time. The time required to resolve a host name.
-	// Use -1 if the timing does not apply to the current request.
-	Connect int `json:"connect,omitempty"`
-	// optional - Time required to create TCP connection. Use -1 if the timing
-	// does not apply to the current request.
-	Send int `json:"send"`
-	// Time required to send HTTP request to the server.
-	Wait int `json:"wait"`
-	// Waiting for a response from the server.
-	Receive int `json:"receive"`
-	// Time required to read entire response from the server (or cache).
-	Ssl int `json:"ssl,omitempty"`
-	// optional (new in 1.2) - Time required for SSL/TLS negotiation. If this
-	// field is defined then the time is also included in the connect field (to
-	// ensure backward compatibility with HAR 1.1). Use -1 if the timing does not
-	// apply to the current request.
-	Comment string `json:"comment,omitempty"`
-	// optional (new in 1.2) - A comment provided by the user or the application.
-}
-
-// TestResult contains results for an individual HTTP request
-type TestResult struct {
-	URL       string    `json:"url"`
-	Status    int       `json:"status"` // 200, 500, etc.
-	StartTime time.Time `json:"startTime"`
-	EndTime   time.Time `json:"endTime"`
-	Latency   int       `json:"latency"` // milliseconds
-	Method    string    `json:"method"`
-	HarFile   string    `json:"harfile"`
-}
-
-// aliases for martian lib compatibility
-
-type Header = NVP
-type QueryString = NVP
-type Param = PostParam
-type Timings = PageTimings

agent/pkg/har/types_test.go

@@ -1,38 +0,0 @@
-package har
-
-import "testing"
-
-func TestContentEncoded(t *testing.T) {
-	testCases := []struct {
-		text        string
-		isBinary    bool
-		expectedStr string
-		binaryLen   int
-	}{
-		{"not-base64", false, "not-base64", 10},
-		{"dGVzdA==", false, "test", 4},
-		{"test", true, "\f@A", 3}, // valid UTF-8 with some non-printable chars
-		{"IsDggPCAgPiAgID8gICAgN/vv/e/v/u/v7/9v7+/vyIKIu+3kO+3ke+3ku+3k++3lO+3le+3lu+3l++3mO+3me+3mu+3m++3nO+3ne+3nu+3n++3oO+3oe+3ou+3o++3pO+3pe+3pu+3p++3qO+3qe+3qu+3q++3rO+3re+3ru+3ryIK", true, "test", 132}, // invalid UTF-8 (thus binary), taken from https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
-	}
-
-	for _, tc := range testCases {
-		c := Content{
-			Encoding: "base64",
-			Text:     tc.text,
-		}
-		isBinary, asBytes, asString := c.B64Decoded()
-		_ = asBytes
-
-		if tc.isBinary != isBinary {
-			t.Errorf("Binary flag mismatch: %t != %t", tc.isBinary, isBinary)
-		}
-
-		if !isBinary && tc.expectedStr != asString {
-			t.Errorf("Decode value mismatch: %s != %s", tc.expectedStr, asString)
-		}
-
-		if tc.binaryLen != len(asBytes) {
-			t.Errorf("Binary len mismatch: %d != %d", tc.binaryLen, len(asBytes))
-		}
-	}
-}

agent/pkg/holder/main.go

@@ -1,6 +1,6 @@
 package holder
 
-import "github.com/up9inc/mizu/agent/pkg/resolver"
+import "mizuserver/pkg/resolver"
 
 var k8sResolver *resolver.Resolver
 
@@ -11,3 +11,4 @@ func SetResolver(param *resolver.Resolver) {
 func GetResolver() *resolver.Resolver {
 	return k8sResolver
 }
+

agent/pkg/middlewares/cors.go

@@ -7,7 +7,7 @@ func CORSMiddleware() gin.HandlerFunc {
 		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
 		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
 		c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With, x-session-token")
-		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE")
+		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT")
 
 		if c.Request.Method == "OPTIONS" {
 			c.AbortWithStatus(204)

agent/pkg/middlewares/requiresAuth.go

@@ -0,0 +1,49 @@
+package middlewares
+
+import (
+	"mizuserver/pkg/config"
+	"mizuserver/pkg/providers"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/patrickmn/go-cache"
+	"github.com/up9inc/mizu/shared/logger"
+)
+
+const cachedValidTokensRetainmentTime = time.Minute * 1
+
+var cachedValidTokens = cache.New(cachedValidTokensRetainmentTime, cachedValidTokensRetainmentTime)
+
+func RequiresAuth() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// auth is irrelevant for ephermeral mizu
+		if !config.Config.StandaloneMode {
+			c.Next()
+			return
+		}
+
+		token := c.GetHeader("x-session-token")
+		if token == "" {
+			c.AbortWithStatusJSON(401, gin.H{"error": "token header is empty"})
+			return
+		}
+
+		if _, isTokenCached := cachedValidTokens.Get(token); isTokenCached {
+			c.Next()
+			return
+		}
+
+		if isTokenValid, err := providers.VerifyToken(token, c.Request.Context()); err != nil {
+			logger.Log.Errorf("error verifying token %s", err)
+			c.AbortWithStatusJSON(401, gin.H{"error": "unknown auth error occured"})
+			return
+		} else if !isTokenValid {
+			c.AbortWithStatusJSON(401, gin.H{"error": "invalid token"})
+			return
+		}
+
+		cachedValidTokens.Set(token, true, cachedValidTokensRetainmentTime)
+
+		c.Next()
+	}
+}

agent/pkg/models/models.go

@@ -2,18 +2,26 @@ package models
 
 import (
 	"encoding/json"
+	"mizuserver/pkg/rules"
 
-	"github.com/up9inc/mizu/agent/pkg/har"
-	"github.com/up9inc/mizu/agent/pkg/rules"
 	tapApi "github.com/up9inc/mizu/tap/api"
 
+	"github.com/google/martian/har"
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"
 )
 
+func GetEntry(r *tapApi.Entry, v tapApi.DataUnmarshaler) error {
+	return v.UnmarshalData(r)
+}
+
+type TapConfig struct {
+	TappedNamespaces map[string]bool `json:"tappedNamespaces"`
+}
+
 type EntriesRequest struct {
-	LeftOff   string `form:"leftOff" validate:"required"`
+	LeftOff   int    `form:"leftOff" validate:"required,min=-1"`
 	Direction int    `form:"direction" validate:"required,oneof='1' '-1'"`
 	Query     string `form:"query"`
 	Limit     int    `form:"limit" validate:"required,min=1"`
@@ -34,11 +42,6 @@ type WebSocketEntryMessage struct {
 	Data *tapApi.BaseEntry `json:"data,omitempty"`
 }
 
-type WebSocketFullEntryMessage struct {
-	*shared.WebSocketMessageMetadata
-	Data *tapApi.Entry `json:"data,omitempty"`
-}
-
 type WebSocketTappedEntryMessage struct {
 	*shared.WebSocketMessageMetadata
 	Data *tapApi.OutputChannelItem
@@ -85,16 +88,6 @@ func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntry) ([]byte, error) {
 	return json.Marshal(message)
 }
 
-func CreateFullEntryWebSocketMessage(entry *tapApi.Entry) ([]byte, error) {
-	message := &WebSocketFullEntryMessage{
-		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
-			MessageType: shared.WebSocketMessageTypeFullEntry,
-		},
-		Data: entry,
-	}
-	return json.Marshal(message)
-}
-
 func CreateWebsocketTappedEntryMessage(base *tapApi.OutputChannelItem) ([]byte, error) {
 	message := &WebSocketTappedEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
@@ -105,6 +98,16 @@ func CreateWebsocketTappedEntryMessage(base *tapApi.OutputChannelItem) ([]byte,
 	return json.Marshal(message)
 }
 
+func CreateWebsocketOutboundLinkMessage(base *tap.OutboundLink) ([]byte, error) {
+	message := &WebsocketOutboundLinkMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebsocketMessageTypeOutboundLink,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
 func CreateWebsocketToastMessage(base *ToastMessage) ([]byte, error) {
 	message := &WebSocketToastMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
@@ -160,3 +163,7 @@ func RunValidationRulesState(harEntry har.Entry, service string) (tapApi.Applica
 	statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend)
 	return tapApi.ApplicableRules{Status: statusPolicyToSend, Latency: latency, NumberOfRules: numberOfRules}, resultPolicyToSend, isEnabled
 }
+
+type InstallState struct {
+	Completed bool `json:"completed"`
+}

agent/pkg/oas/counters.go

@@ -1,119 +0,0 @@
-package oas
-
-import (
-	"fmt"
-	"github.com/chanced/openapi"
-	"math"
-	"strings"
-)
-
-type Counter struct {
-	Entries     int     `json:"entries"`
-	Failures    int     `json:"failures"`
-	FirstSeen   float64 `json:"firstSeen"`
-	LastSeen    float64 `json:"lastSeen"`
-	SumRT       float64 `json:"sumRT"`
-	SumDuration float64 `json:"sumDuration"`
-}
-
-func (c *Counter) addEntry(ts float64, rt float64, succ bool, dur float64) {
-	if dur < 0 {
-		panic("Duration cannot be negative")
-	}
-
-	c.Entries += 1
-	c.SumRT += rt
-	c.SumDuration += dur
-	if !succ {
-		c.Failures += 1
-	}
-
-	if c.FirstSeen == 0 {
-		c.FirstSeen = ts
-	} else {
-		c.FirstSeen = math.Min(c.FirstSeen, ts)
-	}
-
-	c.LastSeen = math.Max(c.LastSeen, ts)
-}
-
-func (c *Counter) addOther(other *Counter) {
-	c.Entries += other.Entries
-	c.SumRT += other.SumRT
-	c.Failures += other.Failures
-	c.SumDuration += other.SumDuration
-
-	if c.FirstSeen == 0 {
-		c.FirstSeen = other.FirstSeen
-	} else {
-		c.FirstSeen = math.Min(c.FirstSeen, other.FirstSeen)
-	}
-
-	c.LastSeen = math.Max(c.LastSeen, other.LastSeen)
-}
-
-type CounterMap map[string]*Counter
-
-func (m *CounterMap) addOther(other *CounterMap) {
-	for src, cmap := range *other {
-		if existing, ok := (*m)[src]; ok {
-			existing.addOther(cmap)
-		} else {
-			copied := *cmap
-			(*m)[src] = &copied
-		}
-	}
-}
-
-func setCounterMsgIfOk(oldStr string, cnt *Counter) string {
-	tpl := "Mizu observed %d entries (%d failed), at %.3f hits/s, average response time is %.3f seconds"
-	if oldStr == "" || (strings.HasPrefix(oldStr, "Mizu ") && strings.HasSuffix(oldStr, " seconds")) {
-		return fmt.Sprintf(tpl, cnt.Entries, cnt.Failures, cnt.SumDuration/float64(cnt.Entries), cnt.SumRT/float64(cnt.Entries))
-	}
-	return oldStr
-}
-
-type CounterMaps struct {
-	counterTotal    Counter
-	counterMapTotal CounterMap
-}
-
-func (m *CounterMaps) processOp(opObj *openapi.Operation) error {
-	if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
-		counter := new(Counter)
-		err := opObj.Extensions.DecodeExtension(CountersTotal, counter)
-		if err != nil {
-			return err
-		}
-		m.counterTotal.addOther(counter)
-
-		opObj.Description = setCounterMsgIfOk(opObj.Description, counter)
-	}
-
-	if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
-		counterMap := new(CounterMap)
-		err := opObj.Extensions.DecodeExtension(CountersPerSource, counterMap)
-		if err != nil {
-			return err
-		}
-		m.counterMapTotal.addOther(counterMap)
-	}
-	return nil
-}
-
-func (m *CounterMaps) processOas(oas *openapi.OpenAPI) error {
-	if oas.Extensions == nil {
-		oas.Extensions = openapi.Extensions{}
-	}
-
-	err := oas.Extensions.SetExtension(CountersTotal, m.counterTotal)
-	if err != nil {
-		return err
-	}
-
-	err = oas.Extensions.SetExtension(CountersPerSource, m.counterMapTotal)
-	if err != nil {
-		return nil
-	}
-	return nil
-}

agent/pkg/oas/feeder_test.go

@@ -4,52 +4,35 @@ import (
 	"bufio"
 	"encoding/json"
 	"errors"
-	"fmt"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
 	"io"
 	"io/ioutil"
-	"net/url"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 	"testing"
-
-	"github.com/up9inc/mizu/agent/pkg/har"
-
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 func getFiles(baseDir string) (result []string, err error) {
-	result = make([]string, 0)
+	result = make([]string, 0, 0)
 	logger.Log.Infof("Reading files from tree: %s", baseDir)
 
-	inputs := []string{baseDir}
-
 	// https://yourbasic.org/golang/list-files-in-directory/
-	visitor := func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
+	err = filepath.Walk(baseDir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			ext := strings.ToLower(filepath.Ext(path))
+			if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
+				result = append(result, path)
+			}
 
-		if info.Mode()&os.ModeSymlink != 0 {
-			path, _ = os.Readlink(path)
-			inputs = append(inputs, path)
 			return nil
-		}
-
-		ext := strings.ToLower(filepath.Ext(path))
-		if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
-			result = append(result, path)
-		}
-
-		return nil
-	}
-
-	for len(inputs) > 0 {
-		path := inputs[0]
-		inputs = inputs[1:]
-		err = filepath.Walk(path, visitor)
-	}
+		})
 
 	sort.SliceStable(result, func(i, j int) bool {
 		return fileSize(result[i]) < fileSize(result[j])
@@ -68,42 +51,32 @@ func fileSize(fname string) int64 {
 	return fi.Size()
 }
 
-func feedEntries(fromFiles []string, isSync bool, gen *defaultOasGenerator) (count uint, err error) {
-	badFiles := make([]string, 0)
-	cnt := uint(0)
+func feedEntries(fromFiles []string) (err error) {
 	for _, file := range fromFiles {
 		logger.Log.Info("Processing file: " + file)
 		ext := strings.ToLower(filepath.Ext(file))
-		eCnt := uint(0)
 		switch ext {
 		case ".har":
-			eCnt, err = feedFromHAR(file, isSync, gen)
+			err = feedFromHAR(file)
 			if err != nil {
 				logger.Log.Warning("Failed processing file: " + err.Error())
-				badFiles = append(badFiles, file)
 				continue
 			}
 		case ".ldjson":
-			eCnt, err = feedFromLDJSON(file, isSync, gen)
+			err = feedFromLDJSON(file)
 			if err != nil {
 				logger.Log.Warning("Failed processing file: " + err.Error())
-				badFiles = append(badFiles, file)
 				continue
 			}
 		default:
-			return 0, errors.New("Unsupported file extension: " + ext)
+			return errors.New("Unsupported file extension: " + ext)
 		}
-		cnt += eCnt
 	}
 
-	for _, f := range badFiles {
-		logger.Log.Infof("Bad file: %s", f)
-	}
-
-	return cnt, nil
+	return nil
 }
 
-func feedFromHAR(file string, isSync bool, gen *defaultOasGenerator) (uint, error) {
+func feedFromHAR(file string) error {
 	fd, err := os.Open(file)
 	if err != nil {
 		panic(err)
@@ -113,44 +86,23 @@ func feedFromHAR(file string, isSync bool, gen *defaultOasGenerator) (uint, erro
 
 	data, err := ioutil.ReadAll(fd)
 	if err != nil {
-		return 0, err
+		return err
 	}
 
 	var harDoc har.HAR
 	err = json.Unmarshal(data, &harDoc)
 	if err != nil {
-		return 0, err
+		return err
 	}
 
-	cnt := uint(0)
 	for _, entry := range harDoc.Log.Entries {
-		cnt += 1
-		feedEntry(&entry, "", file, gen, fmt.Sprintf("%024d", cnt))
+		GetOasGeneratorInstance().PushEntry(entry)
 	}
 
-	return cnt, nil
+	return nil
 }
 
-func feedEntry(entry *har.Entry, source string, file string, gen *defaultOasGenerator, cnt string) {
-	entry.Comment = file
-	if entry.Response.Status == 302 {
-		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
-	}
-
-	if strings.Contains(entry.Request.URL, "some") { // for debugging
-		logger.Log.Debugf("Interesting: %s", entry.Request.URL)
-	}
-
-	u, err := url.Parse(entry.Request.URL)
-	if err != nil {
-		logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
-	}
-
-	ews := EntryWithSource{Entry: *entry, Source: source, Destination: u.Host, Id: cnt}
-	gen.handleHARWithSource(&ews)
-}
-
-func feedFromLDJSON(file string, isSync bool, gen *defaultOasGenerator) (uint, error) {
+func feedFromLDJSON(file string) error {
 	fd, err := os.Open(file)
 	if err != nil {
 		panic(err)
@@ -161,9 +113,8 @@ func feedFromLDJSON(file string, isSync bool, gen *defaultOasGenerator) (uint, e
 	reader := bufio.NewReader(fd)
 
 	var meta map[string]interface{}
+
 	buf := strings.Builder{}
-	cnt := uint(0)
-	source := ""
 	for {
 		substr, isPrefix, err := reader.ReadLine()
 		if err == io.EOF {
@@ -181,31 +132,26 @@ func feedFromLDJSON(file string, isSync bool, gen *defaultOasGenerator) (uint, e
 		if meta == nil {
 			err := json.Unmarshal([]byte(line), &meta)
 			if err != nil {
-				return 0, err
-			}
-			if s, ok := meta["_source"]; ok && s != nil {
-				source = s.(string)
+				return err
 			}
 		} else {
 			var entry har.Entry
 			err := json.Unmarshal([]byte(line), &entry)
 			if err != nil {
 				logger.Log.Warningf("Failed decoding entry: %s", line)
-			} else {
-				cnt += 1
-				feedEntry(&entry, source, file, gen, fmt.Sprintf("%024d", cnt))
 			}
+			GetOasGeneratorInstance().PushEntry(&entry)
 		}
 	}
 
-	return cnt, nil
+	return nil
 }
 
 func TestFilesList(t *testing.T) {
 	res, err := getFiles("./test_artifacts/")
 	t.Log(len(res))
 	t.Log(res)
-	if err != nil || len(res) != 3 {
+	if err != nil || len(res) != 2 {
 		t.Logf("Should return 2 files but returned %d", len(res))
 		t.FailNow()
 	}

agent/pkg/oas/gibberish.go

@@ -1,26 +1,21 @@
 package oas
 
 import (
-	"math"
 	"regexp"
 	"strings"
 	"unicode"
 )
 
 var (
+	patBase64   = regexp.MustCompile(`^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$`)
 	patUuid4    = regexp.MustCompile(`(?i)[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`)
 	patEmail    = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`)
-	patLongNum  = regexp.MustCompile(`^\d{3,}$`)
-	patLongNumB = regexp.MustCompile(`[^\d]\d{3,}`)
-	patLongNumA = regexp.MustCompile(`\d{3,}[^\d]`)
+	patHexLower = regexp.MustCompile(`(0x)?[0-9a-f]{6,}`)
+	patHexUpper = regexp.MustCompile(`(0x)?[0-9A-F]{6,}`)
 )
 
 func IsGibberish(str string) bool {
-	if IsVersionString(str) {
-		return false
-	}
-
-	if patEmail.MatchString(str) {
+	if patBase64.MatchString(str) && len(str) > 32 {
 		return true
 	}
 
@@ -28,44 +23,20 @@ func IsGibberish(str string) bool {
 		return true
 	}
 
-	if patLongNum.MatchString(str) || patLongNumB.MatchString(str) || patLongNumA.MatchString(str) {
+	if patEmail.MatchString(str) {
 		return true
 	}
 
-	//alNum := cleanStr(str, isAlNumRune)
-	//alpha := cleanStr(str, isAlphaRune)
-	// noiseAll := isNoisy(alNum)
-	//triAll := isTrigramBad(strings.ToLower(alpha))
-	// _ = noiseAll
-
-	isNotAlNum := func(r rune) bool { return !isAlNumRune(r) }
-	chunks := strings.FieldsFunc(str, isNotAlNum)
-	noisyLen := 0
-	alnumLen := 0
-	for _, chunk := range chunks {
-		alnumLen += len(chunk)
-		noise := isNoisy(chunk)
-		tri := isTrigramBad(strings.ToLower(chunk))
-		if noise || tri {
-			noisyLen += len(chunk)
-		}
+	if patHexLower.MatchString(str) || patHexUpper.MatchString(str) {
+		return true
 	}
 
-	return float64(noisyLen) > 0
+	noise := noiseLevel(str)
+	if noise >= 0.2 {
+		return true
+	}
 
-	//if float64(noisyLen) > 0 {
-	//	return true
-	//}
-
-	//if len(chunks) > 0 && float64(noisyLen) >= float64(alnumLen)/3.0 {
-	//	return true
-	//}
-
-	//if triAll {
-	//return true
-	//}
-
-	// return false
+	return false
 }
 
 func noiseLevel(str string) (score float64) {
@@ -83,21 +54,21 @@ func noiseLevel(str string) (score float64) {
 
 			// upper =>
 			case unicode.IsUpper(prev) && unicode.IsLower(char):
-				score += 0.10
+				score += 0.25
 			case unicode.IsUpper(prev) && unicode.IsDigit(char):
-				score += 0.5
+				score += 0.25
 
 			// lower =>
 			case unicode.IsLower(prev) && unicode.IsUpper(char):
 				score += 0.75
 			case unicode.IsLower(prev) && unicode.IsDigit(char):
-				score += 0.5
+				score += 0.25
 
 			// digit =>
 			case unicode.IsDigit(prev) && unicode.IsUpper(char):
 				score += 0.75
 			case unicode.IsDigit(prev) && unicode.IsLower(char):
-				score += 1.0
+				score += 0.75
 
 			// the rest is 100% noise
 			default:
@@ -107,6 +78,8 @@ func noiseLevel(str string) (score float64) {
 		prev = char
 	}
 
+	score /= cnt // weigh it
+
 	return score
 }
 
@@ -127,59 +100,9 @@ func IsVersionString(component string) bool {
 		}
 	}
 
-	if !hasV && !strings.Contains(component, ".") {
+	if !hasV && strings.Contains(component, ".") {
 		return false
 	}
 
 	return true
 }
-
-func trigramScore(str string) (float64, int) {
-	tgScore := 0.0
-	trigrams := ngrams(str, 3)
-	if len(trigrams) > 0 {
-		for _, trigram := range trigrams {
-			score, found := corpus_trigrams[trigram]
-			if found {
-				tgScore += score
-			}
-		}
-	}
-
-	return tgScore, len(trigrams)
-}
-
-func isTrigramBad(s string) bool {
-	tgScore, cnt := trigramScore(s)
-
-	if cnt > 0 {
-		val := math.Sqrt(tgScore) / float64(cnt)
-		val2 := tgScore / float64(cnt)
-		threshold := 0.005
-		bad := val < threshold
-		threshold2 := math.Log(float64(cnt)-2) * 0.1
-		bad2 := val2 < threshold2
-		return bad && bad2 // TODO: improve this logic to be clearer
-	}
-	return false
-}
-
-func isNoisy(s string) bool {
-	noise := noiseLevel(s)
-
-	if len(s) > 0 {
-		val := (noise * noise) / float64(len(s))
-		threshold := 0.6
-		bad := val > threshold
-		return bad
-	}
-	return false
-}
-
-func ngrams(s string, n int) []string {
-	result := make([]string, 0)
-	for i := 0; i < len(s)-n+1; i++ {
-		result = append(result, s[i:i+n])
-	}
-	return result
-}

agent/pkg/oas/gibberish_test.go

@@ -1,95 +1,17 @@
 package oas
 
-import (
-	"testing"
-)
+import "testing"
 
 func TestNegative(t *testing.T) {
 	cases := []string{
 		"",
-		"{}",
-		"0.0.29",
-		"0.1",
-		"1.0",
-		"1.0.0",
-		"2.1.73",
-		"abTestV2",
-		"actionText,setName,setAttribute,save,ignore,onEnd,getContext,end,get",
-		"AddUserGroupLink",
-		"advert-management.adBlockerMessage.html",
-		"agents.author.1.json",
-		"animated-gif",
 		"b", // can be valid hexadecimal
-		"big-danger-coronavirus-panic-greater-crisis",
-		"breakout-box",
-		"callback",
-		"core.algorithm_execution.view",
-		"core.devices.view",
-		"data.json",
-		"dialog.overlay.infinity.json",
-		"domain-input",
-		"embeddable", // lul, it's a valid HEX!
-		"embeddable_blip",
-		"E PLURIBUS UNUM",
-		"etc",
-		"eu-central-1a",
-		"fcgi-bin",
-		"footer.include.html",
-		"fullHashes:find",
-		"generate-feed",
-		"GetAds",
-		"GetCart",
 		"GetUniversalVariableUser",
-		"github-audit-exports",
-		"g.js",
-		"g.pixel",
-		".html",
-		"Hugo Michiels",
-		"image.sbix",
-		"index.html",
-		"iPad",
-		"Joanna Mazewski",
-		"LibGit2Sharp",
-		"Michael_Vaughan1.png",
-		"New RSS feed has been generated",
-		"nick-clegg",
-		"opt-out",
-		"pixel_details.html",
-		"post.json",
-		"profile-method-info",
-		"project-id",
-		"publisha.1.json",
-		"publish_and_moderate",
-		"Ronna McDaniel",
-		"rtb-h",
+		"callback",
 		"runs",
-		"sign-up",
-		"some-uuid-maybe",
-		"stable-4.0-version.json",
+		"tcfv2",
 		"StartUpCheckout",
-		"Steve Flunk",
-		"sync_a9",
-		"Ted Cruz",
-		"test.png",
-		"token",
-		"ToList",
-		"v2.1.3",
-		"VersionCheck.php",
-		"v Rusiji",
-		"Walnut St",
-		"web_widget",
-		"zoom_in.cur",
-		"xray",
-		"web",
-		"vipbets1",
-		"trcc",
-		"fbpixel",
-
-		// TODO below
-		// "tcfv2",
-		// "Matt-cartoon-255x206px-small.png",
-		// "TheTelegraph_portal_white-320-small.png",
-		// "testdata-10kB.js",
+		"GetCart",
 	}
 
 	for _, str := range cases {
@@ -101,92 +23,44 @@ func TestNegative(t *testing.T) {
 
 func TestPositive(t *testing.T) {
 	cases := []string{
-		"0a0d0174-b338-4520-a1c3-24f7e3d5ec50.html",
-		"1024807212418223",
-		"11ca096cbc224a67360493d44a9903",
-		"1553183382779",
-		"1554507871",
-		"19180481",
-		"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
+		"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
+		"952bea17-3776-11ea-9341-42010a84012a",
 		"456795af-b48f-4a8d-9b37-3e932622c2f0",
-		"601a2bdcc5b69137248ddbbf",
-		"60fe9aaeaefe2400012df94f",
+		"0a0d0174-b338-4520-a1c3-24f7e3d5ec50.html",
+		"6120c057c7a97b03f6986f1b",
 		"610bc3fd5a77a7fa25033fb0",
 		"610bd0315a77a7fa25034368",
 		"610bd0315a77a7fa25034368zh",
-		"6120c057c7a97b03f6986f1b",
 		"710a462e",
-		"730970532670-compute@developer.gserviceaccount.com",
-		"819db2242a648b305395537022523d65",
-		"952bea17-3776-11ea-9341-42010a84012a",
-		"a3226860758.html",
-		"AAAA028295945",
-		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4",
-		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
-		"bnjksfd897345nl098asd53412kl98",
-		"c738338322370b47a79251f7510dd",                        // prefixed hex
-		"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
-		"css/login.0f48c49a34eb53ea4623.min.css",
-		"d_fLLxlhzDilixeBEimaZ5",
-		"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
-		"e8782afc112720300c049ff124434b79",
-		"fb6cjraf9cejut2a",
-		"i-0236530c66ed02200",
-		"JEHJW4BKVFDRTMTUQLHKK5WVAU",
+		"1554507871",
+		"qwerqwerasdfqwer@protonmai.com",
 		"john.dow.1981@protonmail.com",
-		"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
-		"MNUTGVFMGLEMFTBH0XSE5E02F6J2DS",
-		"n63nd45qsj",
-		"n9z9QGNiz",
-		"NC4WTmcy",
-		"proxy.3d2100fd7107262ecb55ce6847f01fa5.html",
+		"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
+		"11ca096cbc224a67360493d44a9903",
+		"c738338322370b47a79251f7510dd", // prefixed hex
 		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0FgAFKkOhcHdFWzL1ZYggtwBgiB3LSoele9o3ZqFh7iCBhHbVLAnMuJ0HF8hEw7UKecE6wd-MBXgeRMdubGydhAMZSmuUjRpqplML40bmrb8VjJKNZswD1Cg",
 		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0rG4sjLa_KVLlww5WEDJ__30J15en-K_6Y68jb_rU93e2TFY6fb0MYiQ1UrLNMQufqODHZUl39Lo6cXAOVOThjAMZSmuVH7n85JOYSCgzpvowMAVueGG0Xxg",
-		"qwerqwerasdfqwer@protonmai.com",
+		"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
+		"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
+		"730970532670-compute@developer.gserviceaccount.com",
+		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
+		"AAAA028295945",
+		"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
+		"n63nd45qsj",
+		"n9z9QGNiz",
+		"proxy.3d2100fd7107262ecb55ce6847f01fa5.html",
 		"r-ext-5579e00a95c90",
 		"r-ext-5579e8b12f11e",
 		"r-v4-5c92513c9e04a",
 		"r-v4-5c92513c9e04a-0-c.monitoring.json",
 		"segments-1563566437171.639994",
-		"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
-		"sp_dxJTfx11_576742227280287872",
-		"sp_NnUPB5wj_601a2bdcc5b69137248ddbbf",
-		"sp_NxITuoE4_premiumchron-article-14302157_c_ryGQBs_r_yIWvwP",
 		"t_52d94268-8810-4a7e-ba87-ffd657a6752f",
 		"timeouts-1563566437171.639994",
-		"u_YPF3GsGKMo02",
-
-		"0000000000 65535 f",
-		"0000000178 00000 n",
-		"0-10000",
-		"01526123,",
-		"0,18168,183955,3,4,1151616,5663,731,223,5104,207,3204,10,1051,175,364,1435,4,60,576,241,383,246,5,1102",
-		"05/10/2020",
-		"14336456724940333",
-		"fb6cjraf9cejut2a",
-		"JEHJW4BKVFDRTMTUQLHKK5WVAU",
 
 		// TODO
-		/*
-			"0,20",
-			"0.001",
-			"YISAtiX1",
-			"Fxvd1timk", // questionable
-			"B4GCSkORAJs",
-			"D_4EDAqenHQ",
-			"EICJp29EGOk",
-			"Fxvd1timk",
-			"GTqMZELYfQQ",
-			"GZPTpLPEGmwHGWPC",
-			"_HChnE9NDPY",
-			"NwhjgIWHgGg",
-			"production/tsbqksph4xswqjexfbec",
-			"p/u/bguhrxupr23mw3nwxcrw",
-			"nRSNapbJZnc",
-			"zgfpbtolciznub5egzxk",
-			"zufnu7aimadua9wrgwwo",
-			"zznto1jzch9yjsbtbrul",
-		*/
+		// "fb6cjraf9cejut2a",
+		// "Fxvd1timk", // questionable
+		// "JEHJW4BKVFDRTMTUQLHKK5WVAU",
 	}
 
 	for _, str := range cases {
@@ -195,18 +69,3 @@ func TestPositive(t *testing.T) {
 		}
 	}
 }
-
-func TestVersionStrings(t *testing.T) {
-	cases := []string{
-		"1.0",
-		"1.0.0",
-		"v2.1.3",
-		"2.1.73",
-	}
-
-	for _, str := range cases {
-		if !IsVersionString(str) {
-			t.Errorf("Mistakenly false: %s", str)
-		}
-	}
-}

agent/pkg/oas/ignores.go

@@ -12,26 +12,24 @@ var ignoredHeaders = []string{
 	"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-type", "cookie",
 	"date", "dnt", "expect", "forwarded", "from", "front-end-https", "host", "http2-settings",
 	"max-forwards", "origin", "pragma", "proxy-authorization", "proxy-connection", "range", "referer",
-	"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests", "x-download-options",
-	"server", "user-agent", "via", "warning", "strict-transport-security", "x-permitted-cross-domain-policies",
-	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data", "x-dns-prefetch-control",
+	"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests",
+	"server", "user-agent", "via", "warning", "strict-transport-security",
+	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data",
 	"x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh",
 	"x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection",
-	"x-wap-profile", "x-scheme", "status", "x-cache", "x-application-context", "retry-after",
-	"newrelic", "x-cloud-trace-context", "sentry-trace", "x-cache-hits", "x-served-by", "x-span-name",
-	"expires", "set-cookie", "p3p", "content-security-policy", "content-security-policy-report-only",
-	"last-modified", "content-language", "x-varnish", "true-client-ip", "akamai-origin-hop",
+	"x-wap-profile", "x-scheme",
+	"newrelic", "x-cloud-trace-context", "sentry-trace",
+	"expires", "set-cookie", "p3p", "location", "content-security-policy", "content-security-policy-report-only",
+	"last-modified", "content-language",
 	"keep-alive", "etag", "alt-svc", "x-csrf-token", "x-ua-compatible", "vary", "x-powered-by",
-	"age", "allow", "www-authenticate", "expect-ct", "timing-allow-origin", "referrer-policy",
-	"x-aspnet-version", "x-aspnetmvc-version", "x-timer", "x-abuse-info", "x-mod-pagespeed",
-	"duration_ms", // UP9 custom
+	"age", "allow", "www-authenticate",
 }
 
 var ignoredHeaderPrefixes = []string{
 	":", "accept-", "access-control-", "if-", "sec-", "grpc-",
-	"x-forwarded-", "x-original-", "cf-",
+	"x-forwarded-", "x-original-",
 	"x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-",
-	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-", "x-goog-",
+	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-",
 }
 
 func isCtypeIgnored(ctype string) bool {

agent/pkg/oas/oas-samples/carts.json

@@ -1,96 +0,0 @@
-{
-  "openapi": "3.1.0",
-  "info": {
-    "title": "http://carts",
-    "description": "Mizu observed 3 entries (0 failed), at 2.287 hits/s, average response time is 0.017 seconds",
-    "version": "1.0"
-  },
-  "servers": [
-    {
-      "url": "http://carts"
-    }
-  ],
-  "paths": {
-    "/carts/{cartId}/items": {
-      "get": {
-        "summary": "/carts/{cartId}/items",
-        "description": "Mizu observed 3 entries (0 failed), at 2.287 hits/s, average response time is 0.017 seconds",
-        "operationId": "84c9b926-1f73-4ab4-b381-3c124528959f",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": [
-                  {
-                    "id": "60fe98fb86c0fc000869a90c",
-                    "itemId": "3395a43e-2d88-40de-b95f-e00e1502085b",
-                    "quantity": 1,
-                    "unitPrice": 18
-                  }
-                ],
-                "x-sample-entry": "000000000000000000000010"
-              }
-            },
-            "x-sample-entry": "000000000000000000000010"
-          }
-        },
-        "x-counters-per-source": {
-          "some-source": {
-            "entries": 3,
-            "failures": 0,
-            "firstSeen": 1627298058.3798368,
-            "lastSeen": 1627298065.2397773,
-            "sumRT": 0.05,
-            "sumDuration": 6.859940528869629
-          }
-        },
-        "x-counters-total": {
-          "entries": 3,
-          "failures": 0,
-          "firstSeen": 1627298058.3798368,
-          "lastSeen": 1627298065.2397773,
-          "sumRT": 0.05,
-          "sumDuration": 6.859940528869629
-        },
-        "x-last-seen-ts": 1627298065.2397773,
-        "x-sample-entry": "000000000000000000000010"
-      },
-      "parameters": [
-        {
-          "name": "cartId",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "examples": {
-            "example #0": {
-              "value": "mHK0P7zTktmV1zv57iWAvCTd43FFMHap"
-            }
-          },
-          "x-sample-entry": "000000000000000000000010"
-        }
-      ]
-    }
-  },
-  "x-counters-per-source": {
-    "some-source": {
-      "entries": 3,
-      "failures": 0,
-      "firstSeen": 1627298058.3798368,
-      "lastSeen": 1627298065.2397773,
-      "sumRT": 0.05,
-      "sumDuration": 6.859940528869629
-    }
-  },
-  "x-counters-total": {
-    "entries": 3,
-    "failures": 0,
-    "firstSeen": 1627298058.3798368,
-    "lastSeen": 1627298065.2397773,
-    "sumRT": 0.05,
-    "sumDuration": 6.859940528869629
-  }
-}

agent/pkg/oas/oas-samples/catalogue.json

@@ -1,485 +0,0 @@
-{
-  "openapi": "3.1.0",
-  "info": {
-    "title": "Preloaded",
-    "description": "Test file for loading pre-existing OAS",
-    "version": "0.1"
-  },
-  "paths": {
-    "/catalogue": {
-      "get": {
-        "tags": [
-          "catalogue"
-        ],
-        "summary": "/catalogue",
-        "description": "Mizu observed 3 entries (0 failed), at 2.647 hits/s, average response time is 0.008 seconds",
-        "operationId": "dd6c3dbe-6b6b-4ddd-baed-757e237ddb8a",
-        "parameters": [
-          {
-            "name": "page",
-            "in": "query",
-            "required": false,
-            "style": "form",
-            "schema": {
-              "type": "string"
-            },
-            "examples": {
-              "example #0": {
-                "value": "1"
-              }
-            },
-            "x-sample-entry": "000000000000000000000002"
-          },
-          {
-            "name": "size",
-            "in": "query",
-            "required": true,
-            "style": "form",
-            "schema": {
-              "type": "string"
-            },
-            "examples": {
-              "example #0": {
-                "value": "6"
-              },
-              "example #1": {
-                "value": "3"
-              },
-              "example #2": {
-                "value": "5"
-              }
-            },
-            "x-sample-entry": "000000000000000000000011"
-          },
-          {
-            "name": "tags",
-            "in": "query",
-            "required": false,
-            "style": "form",
-            "schema": {
-              "type": "string"
-            },
-            "examples": {
-              "example #0": {
-                "value": ""
-              },
-              "example #1": {
-                "value": "blue"
-              }
-            },
-            "x-sample-entry": "000000000000000000000007"
-          },
-          {
-            "name": "sort",
-            "in": "query",
-            "required": false,
-            "style": "form",
-            "schema": {
-              "type": "string"
-            },
-            "examples": {
-              "example #0": {
-                "value": "id"
-              }
-            },
-            "x-sample-entry": "000000000000000000000007"
-          }
-        ],
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": [
-                  {
-                    "count": 1,
-                    "description": "Socks fit for a Messiah. You too can experience walking in water with these special edition beauties. Each hole is lovingly proggled to leave smooth edges. The only sock approved by a higher power.",
-                    "id": "03fef6ac-1896-4ce8-bd69-b798f85c6e0b",
-                    "imageUrl": [
-                      "/catalogue/images/holy_1.jpeg",
-                      "/catalogue/images/holy_2.jpeg"
-                    ],
-                    "name": "Holy",
-                    "price": 99.99,
-                    "tag": [
-                      "action",
-                      "magic"
-                    ]
-                  },
-                  {
-                    "count": 438,
-                    "description": "proident occaecat irure et excepteur labore minim nisi amet irure",
-                    "id": "3395a43e-2d88-40de-b95f-e00e1502085b",
-                    "imageUrl": [
-                      "/catalogue/images/colourful_socks.jpg",
-                      "/catalogue/images/colourful_socks.jpg"
-                    ],
-                    "name": "Colourful",
-                    "price": 18,
-                    "tag": [
-                      "brown",
-                      "blue"
-                    ]
-                  },
-                  {
-                    "count": 820,
-                    "description": "Ready for action. Engineers: be ready to smash that next bug! Be ready, with these super-action-sport-masterpieces. This particular engineer was chased away from the office with a stick.",
-                    "id": "510a0d7e-8e83-4193-b483-e27e09ddc34d",
-                    "imageUrl": [
-                      "/catalogue/images/puma_1.jpeg",
-                      "/catalogue/images/puma_2.jpeg"
-                    ],
-                    "name": "SuperSport XL",
-                    "price": 15,
-                    "tag": [
-                      "sport",
-                      "formal",
-                      "black"
-                    ]
-                  },
-                  {
-                    "count": 738,
-                    "description": "A mature sock, crossed, with an air of nonchalance.",
-                    "id": "808a2de1-1aaa-4c25-a9b9-6612e8f29a38",
-                    "imageUrl": [
-                      "/catalogue/images/cross_1.jpeg",
-                      "/catalogue/images/cross_2.jpeg"
-                    ],
-                    "name": "Crossed",
-                    "price": 17.32,
-                    "tag": [
-                      "blue",
-                      "action",
-                      "red",
-                      "formal"
-                    ]
-                  },
-                  {
-                    "count": 808,
-                    "description": "enim officia aliqua excepteur esse deserunt quis aliquip nostrud anim",
-                    "id": "819e1fbf-8b7e-4f6d-811f-693534916a8b",
-                    "imageUrl": [
-                      "/catalogue/images/WAT.jpg",
-                      "/catalogue/images/WAT2.jpg"
-                    ],
-                    "name": "Figueroa",
-                    "price": 14,
-                    "tag": [
-                      "green",
-                      "formal",
-                      "blue"
-                    ]
-                  },
-                  {
-                    "count": 175,
-                    "description": "consequat amet cupidatat minim laborum tempor elit ex consequat in",
-                    "id": "837ab141-399e-4c1f-9abc-bace40296bac",
-                    "imageUrl": [
-                      "/catalogue/images/catsocks.jpg",
-                      "/catalogue/images/catsocks2.jpg"
-                    ],
-                    "name": "Cat socks",
-                    "price": 15,
-                    "tag": [
-                      "brown",
-                      "formal",
-                      "green"
-                    ]
-                  }
-                ],
-                "x-sample-entry": "000000000000000000000011"
-              }
-            },
-            "x-sample-entry": "000000000000000000000011"
-          }
-        },
-        "x-counters-per-source": {
-          "some-source": {
-            "entries": 3,
-            "failures": 0,
-            "firstSeen": 1627298057.7849188,
-            "lastSeen": 1627298065.7258668,
-            "sumRT": 0.024999999999999998,
-            "sumDuration": 7.940948009490967
-          }
-        },
-        "x-counters-total": {
-          "entries": 3,
-          "failures": 0,
-          "firstSeen": 1627298057.7849188,
-          "lastSeen": 1627298065.7258668,
-          "sumRT": 0.024999999999999998,
-          "sumDuration": 7.940948009490967
-        },
-        "x-last-seen-ts": 1627298065.7258668,
-        "x-sample-entry": "000000000000000000000011"
-      }
-    },
-    "/catalogue/size": {
-      "get": {
-        "tags": [
-          "catalogue"
-        ],
-        "summary": "/catalogue/size",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.013 seconds",
-        "operationId": "2315e69d-9d66-48cf-b3d3-fec9c30bd28b",
-        "parameters": [
-          {
-            "name": "tags",
-            "in": "query",
-            "required": true,
-            "style": "form",
-            "schema": {
-              "type": "string"
-            },
-            "examples": {
-              "example #0": {
-                "value": ""
-              }
-            },
-            "x-sample-entry": "000000000000000000000001"
-          },
-          {
-            "name": "x-some",
-            "in": "header",
-            "required": true,
-            "style": "simple",
-            "schema": {
-              "type": "string"
-            },
-            "examples": {
-              "example #0": {
-                "value": "demo val"
-              }
-            },
-            "x-sample-entry": "000000000000000000000001"
-          }
-        ],
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": {
-                  "err": null,
-                  "size": 9
-                },
-                "x-sample-entry": "000000000000000000000001"
-              }
-            },
-            "x-sample-entry": "000000000000000000000001"
-          }
-        },
-        "x-counters-per-source": {
-          "some-source": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1627298057.7841518,
-            "lastSeen": 1627298057.7841518,
-            "sumRT": 0.013,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1627298057.7841518,
-          "lastSeen": 1627298057.7841518,
-          "sumRT": 0.013,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1627298057.7841518,
-        "x-sample-entry": "000000000000000000000001"
-      }
-    },
-    "/catalogue/{id}": {
-      "get": {
-        "tags": [
-          "catalogue"
-        ],
-        "summary": "/catalogue/{id}",
-        "description": "Mizu observed 4 entries (0 failed), at 1.899 hits/s, average response time is 0.003 seconds",
-        "parameters": [
-          {
-            "name": "non-required-header",
-            "in": "header",
-            "required": false,
-            "style": "simple",
-            "schema": {
-              "type": "string"
-            },
-            "example": "some-uuid-maybe"
-          },
-          {
-            "name": "x-some",
-            "in": "header",
-            "required": false,
-            "style": "simple",
-            "schema": {
-              "type": "string"
-            },
-            "examples": {
-              "example #0": {
-                "value": "demoval"
-              }
-            },
-            "x-sample-entry": "000000000000000000000004"
-          }
-        ],
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": {
-                  "count": 438,
-                  "description": "proident occaecat irure et excepteur labore minim nisi amet irure",
-                  "id": "3395a43e-2d88-40de-b95f-e00e1502085b",
-                  "imageUrl": [
-                    "/catalogue/images/colourful_socks.jpg",
-                    "/catalogue/images/colourful_socks.jpg"
-                  ],
-                  "name": "Colourful",
-                  "price": 18,
-                  "tag": [
-                    "brown",
-                    "blue"
-                  ]
-                },
-                "x-sample-entry": "000000000000000000000012"
-              }
-            },
-            "x-sample-entry": "000000000000000000000012"
-          }
-        },
-        "x-counters-per-source": {
-          "some-source": {
-            "entries": 4,
-            "failures": 0,
-            "firstSeen": 1627298058.1315014,
-            "lastSeen": 1627298065.7293031,
-            "sumRT": 0.013999999999999999,
-            "sumDuration": 7.597801685333252
-          }
-        },
-        "x-counters-total": {
-          "entries": 4,
-          "failures": 0,
-          "firstSeen": 1627298058.1315014,
-          "lastSeen": 1627298065.7293031,
-          "sumRT": 0.013999999999999999,
-          "sumDuration": 7.597801685333252
-        },
-        "x-last-seen-ts": 1627298065.7293031,
-        "x-sample-entry": "000000000000000000000012"
-      },
-      "parameters": [
-        {
-          "name": "id",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "examples": {
-            "example #0": {
-              "value": "3395a43e-2d88-40de-b95f-e00e1502085b"
-            },
-            "example #1": {
-              "value": "808a2de1-1aaa-4c25-a9b9-6612e8f29a38"
-            }
-          },
-          "example": "some-uuid-maybe",
-          "x-sample-entry": "000000000000000000000012"
-        }
-      ]
-    },
-    "/catalogue/{id}/details": {
-      "parameters": [
-        {
-          "name": "id",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "example": "some-uuid-maybe"
-        }
-      ]
-    },
-    "/tags": {
-      "get": {
-        "summary": "/tags",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.007 seconds",
-        "operationId": "c4d7d0ed-1a78-4370-a049-efe3abc631a6",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": {
-                  "err": null,
-                  "tags": [
-                    "brown",
-                    "geek",
-                    "formal",
-                    "blue",
-                    "skin",
-                    "red",
-                    "action",
-                    "sport",
-                    "black",
-                    "magic",
-                    "green"
-                  ]
-                },
-                "x-sample-entry": "000000000000000000000003"
-              }
-            },
-            "x-sample-entry": "000000000000000000000003"
-          }
-        },
-        "x-counters-per-source": {
-          "some-source": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1627298057.7841816,
-            "lastSeen": 1627298057.7841816,
-            "sumRT": 0.007,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1627298057.7841816,
-          "lastSeen": 1627298057.7841816,
-          "sumRT": 0.007,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1627298057.7841816,
-        "x-sample-entry": "000000000000000000000003"
-      }
-    }
-  },
-  "x-counters-per-source": {
-    "some-source": {
-      "entries": 9,
-      "failures": 0,
-      "firstSeen": 1627298057.7841518,
-      "lastSeen": 1627298065.7293031,
-      "sumRT": 0.05899999999999999,
-      "sumDuration": 15.538749694824219
-    }
-  },
-  "x-counters-total": {
-    "entries": 9,
-    "failures": 0,
-    "firstSeen": 1627298057.7841518,
-    "lastSeen": 1627298065.7293031,
-    "sumRT": 0.05899999999999999,
-    "sumDuration": 15.538749694824219
-  }
-}

agent/pkg/oas/oas-samples/httpbin.org.json

@@ -1,897 +0,0 @@
-{
-  "openapi": "3.1.0",
-  "info": {
-    "title": "https://httpbin.org",
-    "description": "Mizu observed 19 entries (0 failed), at 0.106 hits/s, average response time is 0.172 seconds",
-    "version": "1.0"
-  },
-  "servers": [
-    {
-      "url": "https://httpbin.org"
-    }
-  ],
-  "paths": {
-    "/appears-once": {
-      "get": {
-        "summary": "/appears-once",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
-        "operationId": "2d34623e-fde8-4720-8390-9a7439051755",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": null,
-                "x-sample-entry": "000000000000000000000004"
-              }
-            },
-            "x-sample-entry": "000000000000000000000004"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750580.0471218,
-            "lastSeen": 1567750580.0471218,
-            "sumRT": 0.63,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750580.0471218,
-          "lastSeen": 1567750580.0471218,
-          "sumRT": 0.63,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750580.0471218,
-        "x-sample-entry": "000000000000000000000004"
-      }
-    },
-    "/appears-twice": {
-      "get": {
-        "summary": "/appears-twice",
-        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds",
-        "operationId": "9c5330f3-8062-468b-b5a3-df1ad82b4846",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": null,
-                "x-sample-entry": "000000000000000000000006"
-              }
-            },
-            "x-sample-entry": "000000000000000000000006"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 2,
-            "failures": 0,
-            "firstSeen": 1567750580.7471218,
-            "lastSeen": 1567750581.7471218,
-            "sumRT": 1.26,
-            "sumDuration": 1
-          }
-        },
-        "x-counters-total": {
-          "entries": 2,
-          "failures": 0,
-          "firstSeen": 1567750580.7471218,
-          "lastSeen": 1567750581.7471218,
-          "sumRT": 1.26,
-          "sumDuration": 1
-        },
-        "x-last-seen-ts": 1567750581.7471218,
-        "x-sample-entry": "000000000000000000000006"
-      }
-    },
-    "/body-optional": {
-      "post": {
-        "summary": "/body-optional",
-        "description": "Mizu observed 3 entries (0 failed), at 0.003 hits/s, average response time is 0.001 seconds",
-        "operationId": "34f3d66c-b1f7-4dca-9cab-987fcc8ae472",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000012"
-              }
-            },
-            "x-sample-entry": "000000000000000000000012"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 3,
-            "failures": 0,
-            "firstSeen": 1567750581.7471218,
-            "lastSeen": 1567750581.757122,
-            "sumRT": 0.003,
-            "sumDuration": 0.010000228881835938
-          }
-        },
-        "x-counters-total": {
-          "entries": 3,
-          "failures": 0,
-          "firstSeen": 1567750581.7471218,
-          "lastSeen": 1567750581.757122,
-          "sumRT": 0.003,
-          "sumDuration": 0.010000228881835938
-        },
-        "x-last-seen-ts": 1567750581.757122,
-        "x-sample-entry": "000000000000000000000012",
-        "requestBody": {
-          "description": "Generic request body",
-          "content": {
-            "application/json": {
-              "example": "{\"key\", \"val\"}",
-              "x-sample-entry": "000000000000000000000011"
-            }
-          },
-          "x-sample-entry": "000000000000000000000012"
-        }
-      }
-    },
-    "/body-required": {
-      "post": {
-        "summary": "/body-required",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
-        "operationId": "ff6add53-ab1c-4d4e-b590-0835fa318276",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000013"
-              }
-            },
-            "x-sample-entry": "000000000000000000000013"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750581.757122,
-            "lastSeen": 1567750581.757122,
-            "sumRT": 0.001,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750581.757122,
-          "lastSeen": 1567750581.757122,
-          "sumRT": 0.001,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750581.757122,
-        "x-sample-entry": "000000000000000000000013",
-        "requestBody": {
-          "description": "Generic request body",
-          "content": {
-            "": {
-              "example": "body exists",
-              "x-sample-entry": "000000000000000000000013"
-            }
-          },
-          "required": true,
-          "x-sample-entry": "000000000000000000000013"
-        }
-      }
-    },
-    "/form-multipart": {
-      "post": {
-        "summary": "/form-multipart",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
-        "operationId": "153f0925-9fc7-4e9f-9d33-f1470f25f0f7",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "example": {},
-                "x-sample-entry": "000000000000000000000009"
-              }
-            },
-            "x-sample-entry": "000000000000000000000009"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750582.7471218,
-            "lastSeen": 1567750582.7471218,
-            "sumRT": 0.001,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750582.7471218,
-          "lastSeen": 1567750582.7471218,
-          "sumRT": 0.001,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750582.7471218,
-        "x-sample-entry": "000000000000000000000009",
-        "requestBody": {
-          "description": "Generic request body",
-          "content": {
-            "multipart/form-data": {
-              "schema": {
-                "type": "object",
-                "required": [
-                  "file",
-                  "path"
-                ],
-                "properties": {
-                  "file": {
-                    "type": "string",
-                    "contentMediaType": "application/json",
-                    "examples": [
-                      "{\"functions\": 123}"
-                    ]
-                  },
-                  "path": {
-                    "type": "string",
-                    "examples": [
-                      "/content/components"
-                    ]
-                  }
-                }
-              },
-              "example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n",
-              "x-sample-entry": "000000000000000000000009"
-            }
-          },
-          "required": true,
-          "x-sample-entry": "000000000000000000000009"
-        }
-      }
-    },
-    "/form-urlencoded": {
-      "post": {
-        "summary": "/form-urlencoded",
-        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds",
-        "operationId": "c92189f5-5636-46eb-ac71-92b17941a568",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000008"
-              }
-            },
-            "x-sample-entry": "000000000000000000000008"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 2,
-            "failures": 0,
-            "firstSeen": 1567750580.7471218,
-            "lastSeen": 1567750581.7471218,
-            "sumRT": 0.002,
-            "sumDuration": 1
-          }
-        },
-        "x-counters-total": {
-          "entries": 2,
-          "failures": 0,
-          "firstSeen": 1567750580.7471218,
-          "lastSeen": 1567750581.7471218,
-          "sumRT": 0.002,
-          "sumDuration": 1
-        },
-        "x-last-seen-ts": 1567750581.7471218,
-        "x-sample-entry": "000000000000000000000008",
-        "requestBody": {
-          "description": "Generic request body",
-          "content": {
-            "application/x-www-form-urlencoded": {
-              "schema": {
-                "type": "object",
-                "required": [
-                  "agent-id",
-                  "callback-url",
-                  "token"
-                ],
-                "properties": {
-                  "agent-id": {
-                    "type": "string",
-                    "examples": [
-                      "ade"
-                    ]
-                  },
-                  "callback-url": {
-                    "type": "string",
-                    "examples": [
-                      ""
-                    ]
-                  },
-                  "optional": {
-                    "type": "string",
-                    "examples": [
-                      "another"
-                    ]
-                  },
-                  "token": {
-                    "type": "string",
-                    "examples": [
-                      "sometoken",
-                      "sometoken-second-val"
-                    ]
-                  }
-                }
-              },
-              "example": "agent-id=ade\u0026callback-url=\u0026token=sometoken",
-              "x-sample-entry": "000000000000000000000008"
-            }
-          },
-          "required": true,
-          "x-sample-entry": "000000000000000000000008"
-        }
-      }
-    },
-    "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}": {
-      "get": {
-        "tags": [
-          "param-patterns"
-        ],
-        "summary": "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
-        "operationId": "85270437-7aae-4a5b-b988-3662092463d0",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000014"
-              }
-            },
-            "x-sample-entry": "000000000000000000000014"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750582,
-            "lastSeen": 1567750582,
-            "sumRT": 0.001,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750582,
-          "lastSeen": 1567750582,
-          "sumRT": 0.001,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750582,
-        "x-sample-entry": "000000000000000000000014"
-      },
-      "parameters": [
-        {
-          "name": "prefixgibberishfineId",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "examples": {
-            "example #0": {
-              "value": "234324"
-            }
-          },
-          "x-sample-entry": "000000000000000000000014"
-        }
-      ]
-    },
-    "/param-patterns/{parampatternId}": {
-      "get": {
-        "tags": [
-          "param-patterns"
-        ],
-        "summary": "/param-patterns/{parampatternId}",
-        "description": "Mizu observed 2 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
-        "operationId": "da597734-1cf5-4d3b-917b-6b02dacf7b7b",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000018"
-              }
-            },
-            "x-sample-entry": "000000000000000000000018"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 2,
-            "failures": 0,
-            "firstSeen": 1567750582.000003,
-            "lastSeen": 1567750582.000004,
-            "sumRT": 0.002,
-            "sumDuration": 9.5367431640625e-7
-          }
-        },
-        "x-counters-total": {
-          "entries": 2,
-          "failures": 0,
-          "firstSeen": 1567750582.000003,
-          "lastSeen": 1567750582.000004,
-          "sumRT": 0.002,
-          "sumDuration": 9.5367431640625e-7
-        },
-        "x-last-seen-ts": 1567750582.000004,
-        "x-sample-entry": "000000000000000000000018"
-      },
-      "parameters": [
-        {
-          "name": "parampatternId",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string",
-            "pattern": "^prefix-gibberish-.+"
-          },
-          "examples": {
-            "example #0": {
-              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
-            },
-            "example #1": {
-              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
-            },
-            "example #2": {
-              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
-            },
-            "example #3": {
-              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
-            },
-            "example #4": {
-              "value": "prefix-gibberish-afterwards"
-            }
-          },
-          "x-sample-entry": "000000000000000000000019"
-        }
-      ]
-    },
-    "/param-patterns/{parampatternId}/1": {
-      "get": {
-        "tags": [
-          "param-patterns"
-        ],
-        "summary": "/param-patterns/{parampatternId}/1",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
-        "operationId": "e965a245-9cfc-48ed-94e1-f765eadb3960",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000015"
-              }
-            },
-            "x-sample-entry": "000000000000000000000015"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750582.000001,
-            "lastSeen": 1567750582.000001,
-            "sumRT": 0.001,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750582.000001,
-          "lastSeen": 1567750582.000001,
-          "sumRT": 0.001,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750582.000001,
-        "x-sample-entry": "000000000000000000000015"
-      },
-      "parameters": [
-        {
-          "name": "parampatternId",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string",
-            "pattern": "^prefix-gibberish-.+"
-          },
-          "examples": {
-            "example #0": {
-              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
-            },
-            "example #1": {
-              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
-            },
-            "example #2": {
-              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
-            },
-            "example #3": {
-              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
-            },
-            "example #4": {
-              "value": "prefix-gibberish-afterwards"
-            }
-          },
-          "x-sample-entry": "000000000000000000000019"
-        }
-      ]
-    },
-    "/param-patterns/{parampatternId}/static": {
-      "get": {
-        "tags": [
-          "param-patterns"
-        ],
-        "summary": "/param-patterns/{parampatternId}/static",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
-        "operationId": "7af420dc-f8b7-450f-8f6f-18b039aa3cde",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000016"
-              }
-            },
-            "x-sample-entry": "000000000000000000000016"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750582.000002,
-            "lastSeen": 1567750582.000002,
-            "sumRT": 0.001,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750582.000002,
-          "lastSeen": 1567750582.000002,
-          "sumRT": 0.001,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750582.000002,
-        "x-sample-entry": "000000000000000000000016"
-      },
-      "parameters": [
-        {
-          "name": "parampatternId",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string",
-            "pattern": "^prefix-gibberish-.+"
-          },
-          "examples": {
-            "example #0": {
-              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
-            },
-            "example #1": {
-              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
-            },
-            "example #2": {
-              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
-            },
-            "example #3": {
-              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
-            },
-            "example #4": {
-              "value": "prefix-gibberish-afterwards"
-            }
-          },
-          "x-sample-entry": "000000000000000000000019"
-        }
-      ]
-    },
-    "/param-patterns/{parampatternId}/{param1}": {
-      "get": {
-        "tags": [
-          "param-patterns"
-        ],
-        "summary": "/param-patterns/{parampatternId}/{param1}",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
-        "operationId": "02a1771d-2d50-4a8c-8be2-29c7e59b8435",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000019"
-              }
-            },
-            "x-sample-entry": "000000000000000000000019"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750582.000002,
-            "lastSeen": 1567750582.000002,
-            "sumRT": 0.001,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750582.000002,
-          "lastSeen": 1567750582.000002,
-          "sumRT": 0.001,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750582.000002,
-        "x-sample-entry": "000000000000000000000019"
-      },
-      "parameters": [
-        {
-          "name": "param1",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "examples": {
-            "example #0": {
-              "value": "23421"
-            }
-          },
-          "x-sample-entry": "000000000000000000000019"
-        },
-        {
-          "name": "parampatternId",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string",
-            "pattern": "^prefix-gibberish-.+"
-          },
-          "examples": {
-            "example #0": {
-              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
-            },
-            "example #1": {
-              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
-            },
-            "example #2": {
-              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
-            },
-            "example #3": {
-              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
-            },
-            "example #4": {
-              "value": "prefix-gibberish-afterwards"
-            }
-          },
-          "x-sample-entry": "000000000000000000000019"
-        }
-      ]
-    },
-    "/{Id}": {
-      "get": {
-        "summary": "/{Id}",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
-        "operationId": "77ec4910-d47a-46a5-8234-fb80a11034b4",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": null,
-                "x-sample-entry": "000000000000000000000003"
-              }
-            },
-            "x-sample-entry": "000000000000000000000003"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750579.7471218,
-            "lastSeen": 1567750579.7471218,
-            "sumRT": 0.63,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750579.7471218,
-          "lastSeen": 1567750579.7471218,
-          "sumRT": 0.63,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750579.7471218,
-        "x-sample-entry": "000000000000000000000003"
-      },
-      "parameters": [
-        {
-          "name": "Id",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "examples": {
-            "example #0": {
-              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
-            },
-            "example #1": {
-              "value": "952bea17-3776-11ea-9341-42010a84012a"
-            }
-          },
-          "x-sample-entry": "000000000000000000000003"
-        }
-      ]
-    },
-    "/{Id}/sub1": {
-      "get": {
-        "summary": "/{Id}/sub1",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds",
-        "operationId": "198675eb-9faf-407b-83fa-0483a730bbbe",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "text/html": {
-                "x-sample-entry": "000000000000000000000001"
-              }
-            },
-            "x-sample-entry": "000000000000000000000001"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750483.864529,
-            "lastSeen": 1567750483.864529,
-            "sumRT": 0.111,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750483.864529,
-          "lastSeen": 1567750483.864529,
-          "sumRT": 0.111,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750483.864529,
-        "x-sample-entry": "000000000000000000000001"
-      },
-      "parameters": [
-        {
-          "name": "Id",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "examples": {
-            "example #0": {
-              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
-            },
-            "example #1": {
-              "value": "952bea17-3776-11ea-9341-42010a84012a"
-            }
-          },
-          "x-sample-entry": "000000000000000000000003"
-        }
-      ]
-    },
-    "/{Id}/sub2": {
-      "get": {
-        "summary": "/{Id}/sub2",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
-        "operationId": "31d880f1-152f-4dd6-84a7-463e13b694a5",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": null,
-                "x-sample-entry": "000000000000000000000002"
-              }
-            },
-            "x-sample-entry": "000000000000000000000002"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750578.7471218,
-            "lastSeen": 1567750578.7471218,
-            "sumRT": 0.63,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750578.7471218,
-          "lastSeen": 1567750578.7471218,
-          "sumRT": 0.63,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750578.7471218,
-        "x-sample-entry": "000000000000000000000002"
-      },
-      "parameters": [
-        {
-          "name": "Id",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "examples": {
-            "example #0": {
-              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
-            },
-            "example #1": {
-              "value": "952bea17-3776-11ea-9341-42010a84012a"
-            }
-          },
-          "x-sample-entry": "000000000000000000000003"
-        }
-      ]
-    }
-  },
-  "x-counters-per-source": {
-    "": {
-      "entries": 19,
-      "failures": 0,
-      "firstSeen": 1567750483.864529,
-      "lastSeen": 1567750582.7471218,
-      "sumRT": 3.273999999999999,
-      "sumDuration": 2.0100011825561523
-    }
-  },
-  "x-counters-total": {
-    "entries": 19,
-    "failures": 0,
-    "firstSeen": 1567750483.864529,
-    "lastSeen": 1567750582.7471218,
-    "sumRT": 3.273999999999999,
-    "sumDuration": 2.0100011825561523
-  }
-}

agent/pkg/oas/oas_generator.go

@@ -3,224 +3,105 @@ package oas
 import (
 	"context"
 	"encoding/json"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
 	"net/url"
 	"sync"
-
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/agent/pkg/har"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap/api"
-
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 var (
 	syncOnce sync.Once
-	instance *defaultOasGenerator
+	instance *oasGenerator
 )
 
-type OasGenerator interface {
-	Start(conn *basenine.Connection)
-	Stop()
-	IsStarted() bool
-	GetServiceSpecs() *sync.Map
-	SetEntriesQuery(query string) bool
-}
-
-type defaultOasGenerator struct {
-	started      bool
-	ctx          context.Context
-	cancel       context.CancelFunc
-	serviceSpecs *sync.Map
-	dbConn       *basenine.Connection
-	dbMutex      sync.Mutex
-	entriesQuery string
-}
-
-func GetDefaultOasGeneratorInstance() *defaultOasGenerator {
+func GetOasGeneratorInstance() *oasGenerator {
 	syncOnce.Do(func() {
-		instance = NewDefaultOasGenerator()
-		logger.Log.Debug("OAS Generator Initialized")
+		instance = newOasGenerator()
+		logger.Log.Debug("Oas Generator Initialized")
 	})
 	return instance
 }
 
-func (g *defaultOasGenerator) Start(conn *basenine.Connection) {
+func (g *oasGenerator) Start() {
 	if g.started {
 		return
 	}
-
-	if g.dbConn == nil {
-		if conn == nil {
-			logger.Log.Infof("Creating new DB connection for OAS generator to address %s:%s", shared.BasenineHost, shared.BaseninePort)
-			newConn, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
-			if err != nil {
-				logger.Log.Error("Error connecting to DB for OAS generator, err: %v", err)
-				return
-			}
-
-			conn = newConn
-		}
-
-		g.dbConn = conn
-	}
-
 	ctx, cancel := context.WithCancel(context.Background())
 	g.cancel = cancel
 	g.ctx = ctx
-	g.serviceSpecs = &sync.Map{}
-
+	g.entriesChan = make(chan har.Entry, 100) // buffer up to 100 entries for OAS processing
+	g.ServiceSpecs = &sync.Map{}
 	g.started = true
-
-	go g.runGenerator()
+	go instance.runGeneretor()
 }
 
-func (g *defaultOasGenerator) Stop() {
-	if !g.started {
-		return
-	}
-	g.started = false
-
-	g.cancel()
-	g.reset()
-
-	g.dbMutex.Lock()
-	defer g.dbMutex.Unlock()
-	if g.dbConn != nil {
-		g.dbConn.Close()
-		g.dbConn = nil
-	}
-}
-
-func (g *defaultOasGenerator) IsStarted() bool {
-	return g.started
-}
-
-func (g *defaultOasGenerator) runGenerator() {
-	// Make []byte channels to receive the data and the meta
-	dataChan := make(chan []byte)
-	metaChan := make(chan []byte)
-
-	g.dbMutex.Lock()
-	defer g.dbMutex.Unlock()
-	logger.Log.Infof("Querying DB for OAS generator with query '%s'", g.entriesQuery)
-	if err := g.dbConn.Query(g.entriesQuery, dataChan, metaChan); err != nil {
-		logger.Log.Errorf("Query mode call failed: %v", err)
-	}
-
+func (g *oasGenerator) runGeneretor() {
 	for {
 		select {
 		case <-g.ctx.Done():
 			logger.Log.Infof("OAS Generator was canceled")
-			close(dataChan)
-			close(metaChan)
 			return
 
-		case metaBytes, ok := <-metaChan:
-			if !ok {
-				logger.Log.Infof("OAS Generator - meta channel closed")
-				break
-			}
-			logger.Log.Debugf("Meta: %s", metaBytes)
-
-		case dataBytes, ok := <-dataChan:
+		case entry, ok := <-g.entriesChan:
 			if !ok {
 				logger.Log.Infof("OAS Generator - entries channel closed")
 				break
 			}
-
-			logger.Log.Debugf("Data: %s", dataBytes)
-			e := new(api.Entry)
-			err := json.Unmarshal(dataBytes, e)
+			u, err := url.Parse(entry.Request.URL)
 			if err != nil {
+				logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
+			}
+
+			val, found := g.ServiceSpecs.Load(u.Host)
+			var gen *SpecGen
+			if !found {
+				gen = NewGen(u.Scheme + "://" + u.Host)
+				g.ServiceSpecs.Store(u.Host, gen)
+			} else {
+				gen = val.(*SpecGen)
+			}
+
+			opId, err := gen.feedEntry(entry)
+			if err != nil {
+				txt, suberr := json.Marshal(entry)
+				if suberr == nil {
+					logger.Log.Debugf("Problematic entry: %s", txt)
+				}
+
+				logger.Log.Warningf("Failed processing entry: %s", err)
 				continue
 			}
-			g.handleEntry(e)
+
+			logger.Log.Debugf("Handled entry %s as opId: %s", entry.Request.URL, opId) // TODO: set opId back to entry?
 		}
 	}
 }
 
-func (g *defaultOasGenerator) handleEntry(mizuEntry *api.Entry) {
-	if mizuEntry.Protocol.Name == "http" {
-		entry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
-		if err != nil {
-			logger.Log.Warningf("Failed to turn MizuEntry %d into HAR Entry: %s", mizuEntry.Id, err)
-			return
-		}
-
-		dest := mizuEntry.Destination.Name
-		if dest == "" {
-			dest = mizuEntry.Destination.IP + ":" + mizuEntry.Destination.Port
-		}
-
-		entryWSource := &EntryWithSource{
-			Entry:       *entry,
-			Source:      mizuEntry.Source.Name,
-			Destination: dest,
-			Id:          mizuEntry.Id,
-		}
-
-		g.handleHARWithSource(entryWSource)
-	} else {
-		logger.Log.Debugf("OAS: Unsupported protocol in entry %d: %s", mizuEntry.Id, mizuEntry.Protocol.Name)
-	}
-}
-
-func (g *defaultOasGenerator) handleHARWithSource(entryWSource *EntryWithSource) {
-	entry := entryWSource.Entry
-	gen := g.getGen(entryWSource.Destination, entry.Request.URL)
-
-	opId, err := gen.feedEntry(entryWSource)
-	if err != nil {
-		txt, suberr := json.Marshal(entry)
-		if suberr == nil {
-			logger.Log.Debugf("Problematic entry: %s", txt)
-		}
-
-		logger.Log.Warningf("Failed processing entry %d: %s", entryWSource.Id, err)
+func (g *oasGenerator) PushEntry(entry *har.Entry) {
+	if !g.started {
 		return
 	}
-
-	logger.Log.Debugf("Handled entry %s as opId: %s", entryWSource.Id, opId) // TODO: set opId back to entry?
-}
-
-func (g *defaultOasGenerator) getGen(dest string, urlStr string) *SpecGen {
-	u, err := url.Parse(urlStr)
-	if err != nil {
-		logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", urlStr, err)
+	select {
+	case g.entriesChan <- *entry:
+	default:
+		logger.Log.Warningf("OAS Generator - entry wasn't sent to channel because the channel has no buffer or there is no receiver")
 	}
-
-	val, found := g.serviceSpecs.Load(dest)
-	var gen *SpecGen
-	if !found {
-		gen = NewGen(u.Scheme + "://" + dest)
-		g.serviceSpecs.Store(dest, gen)
-	} else {
-		gen = val.(*SpecGen)
-	}
-	return gen
 }
 
-func (g *defaultOasGenerator) reset() {
-	g.serviceSpecs = &sync.Map{}
-}
-
-func (g *defaultOasGenerator) GetServiceSpecs() *sync.Map {
-	return g.serviceSpecs
-}
-
-func (g *defaultOasGenerator) SetEntriesQuery(query string) bool {
-	changed := g.entriesQuery != query
-	g.entriesQuery = query
-	return changed
-}
-
-func NewDefaultOasGenerator() *defaultOasGenerator {
-	return &defaultOasGenerator{
+func newOasGenerator() *oasGenerator {
+	return &oasGenerator{
 		started:      false,
 		ctx:          nil,
 		cancel:       nil,
-		serviceSpecs: nil,
-		dbConn:       nil,
+		ServiceSpecs: nil,
+		entriesChan:  nil,
 	}
 }
+
+type oasGenerator struct {
+	started      bool
+	ctx          context.Context
+	cancel       context.CancelFunc
+	ServiceSpecs *sync.Map
+	entriesChan  chan har.Entry
+}

agent/pkg/oas/oas_generator_test.go

@@ -1,46 +0,0 @@
-package oas
-
-import (
-	"encoding/json"
-	"github.com/up9inc/mizu/agent/pkg/har"
-	"testing"
-	"time"
-)
-
-func TestOASGen(t *testing.T) {
-	gen := new(defaultOasGenerator)
-
-	e := new(har.Entry)
-	err := json.Unmarshal([]byte(`{"startedDateTime": "20000101","request": {"url": "https://host/path", "method": "GET"}, "response": {"status": 200}}`), e)
-	if err != nil {
-		panic(err)
-	}
-
-	ews := &EntryWithSource{
-		Destination: "some",
-		Entry:       *e,
-	}
-
-	dummyConn := GetFakeDBConn(`{"startedDateTime": "20000101","request": {"url": "https://host/path", "method": "GET"}, "response": {"status": 200}}`)
-	gen.Start(dummyConn)
-	gen.handleHARWithSource(ews)
-	g, ok := gen.serviceSpecs.Load("some")
-	if !ok {
-		panic("Failed")
-	}
-	sg := g.(*SpecGen)
-	spec, err := sg.GetSpec()
-	if err != nil {
-		panic(err)
-	}
-	specText, _ := json.Marshal(spec)
-	t.Log(string(specText))
-
-	if !gen.IsStarted() {
-		t.Errorf("Should be started")
-	}
-
-	time.Sleep(100 * time.Millisecond)
-
-	gen.Stop()
-}

agent/pkg/oas/specgen.go

@@ -1,41 +1,20 @@
 package oas
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"errors"
-	"io"
-	"io/ioutil"
+	"github.com/chanced/openapi"
+	"github.com/google/martian/har"
+	"github.com/google/uuid"
+	"github.com/up9inc/mizu/shared/logger"
 	"mime"
-	"mime/multipart"
-	"net/textproto"
 	"net/url"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
-
-	"github.com/chanced/openapi"
-	"github.com/google/uuid"
-	"github.com/nav-inc/datetime"
-	"github.com/up9inc/mizu/shared/logger"
-
-	"github.com/up9inc/mizu/agent/pkg/har"
-
-	"time"
 )
 
-const LastSeenTS = "x-last-seen-ts"
-const CountersTotal = "x-counters-total"
-const CountersPerSource = "x-counters-per-source"
-const SampleId = "x-sample-entry"
-
-type EntryWithSource struct {
-	Source      string
-	Destination string
-	Entry       har.Entry
-	Id          string
-}
-
 type reqResp struct { // hello, generics in Go
 	Req  *har.Request
 	Resp *har.Response
@@ -52,7 +31,7 @@ func NewGen(server string) *SpecGen {
 	spec.Version = "3.1.0"
 
 	info := openapi.Info{Title: server}
-	info.Version = "1.0"
+	info.Version = "0.0"
 	spec.Info = &info
 	spec.Paths = &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
 
@@ -65,23 +44,17 @@ func NewGen(server string) *SpecGen {
 
 func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
 	g.oas = oas
-	g.tree = new(Node)
 	for pathStr, pathObj := range oas.Paths.Items {
 		pathSplit := strings.Split(string(pathStr), "/")
-		g.tree.getOrSet(pathSplit, pathObj, "")
-
-		// clean "last entry timestamp" markers from the past
-		for _, pathAndOp := range g.tree.listOps() {
-			delete(pathAndOp.op.Extensions, LastSeenTS)
-		}
+		g.tree.getOrSet(pathSplit, pathObj)
 	}
 }
 
-func (g *SpecGen) feedEntry(entryWithSource *EntryWithSource) (string, error) {
+func (g *SpecGen) feedEntry(entry har.Entry) (string, error) {
 	g.lock.Lock()
 	defer g.lock.Unlock()
 
-	opId, err := g.handlePathObj(entryWithSource)
+	opId, err := g.handlePathObj(&entry)
 	if err != nil {
 		return "", err
 	}
@@ -96,23 +69,10 @@ func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
 
 	g.tree.compact()
 
-	counters := CounterMaps{counterTotal: Counter{}, counterMapTotal: CounterMap{}}
-
-	for _, pathAndOp := range g.tree.listOps() {
-		opObj := pathAndOp.op
-		if opObj.Summary == "" {
-			opObj.Summary = pathAndOp.path
+	for _, pathop := range g.tree.listOps() {
+		if pathop.op.Summary == "" {
+			pathop.op.Summary = pathop.path
 		}
-
-		err := counters.processOp(opObj)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	err := counters.processOas(g.oas)
-	if err != nil {
-		return nil, err
 	}
 
 	// put paths back from tree into OAS
@@ -120,8 +80,6 @@ func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
 
 	suggestTags(g.oas)
 
-	g.oas.Info.Description = setCounterMsgIfOk(g.oas.Info.Description, &counters.counterTotal)
-
 	// to make a deep copy, no better idea than marshal+unmarshal
 	specText, err := json.MarshalIndent(g.oas, "", "\t")
 	if err != nil {
@@ -139,7 +97,6 @@ func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
 
 func suggestTags(oas *openapi.OpenAPI) {
 	paths := getPathsKeys(oas.Paths.Items)
-	sort.Strings(paths) // make it stable in case of multiple candidates
 	for len(paths) > 0 {
 		group := make([]string, 0)
 		group = append(group, paths[0])
@@ -169,9 +126,37 @@ func suggestTags(oas *openapi.OpenAPI) {
 				}
 			}
 		}
+
+		//groups[common] = group
 	}
 }
 
+func getSimilarPrefix(strs []string) string {
+	chunked := make([][]string, 0)
+	for _, item := range strs {
+		chunked = append(chunked, strings.Split(item, "/"))
+	}
+
+	cmn := longestCommonXfix(chunked, true)
+	res := make([]string, 0)
+	for _, chunk := range cmn {
+		if chunk != "api" && !IsVersionString(chunk) && !strings.HasPrefix(chunk, "{") {
+			res = append(res, chunk)
+		}
+	}
+	return strings.Join(res[1:], ".")
+}
+
+func deleteFromSlice(s []string, val string) []string {
+	temp := s[:0]
+	for _, x := range s {
+		if x != val {
+			temp = append(temp, x)
+		}
+	}
+	return temp
+}
+
 func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
 	keys := make([]string, len(mymap))
 
@@ -183,8 +168,7 @@ func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
 	return keys
 }
 
-func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error) {
-	entry := entryWithSource.Entry
+func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
 	urlParsed, err := url.Parse(entry.Request.URL)
 	if err != nil {
 		return "", err
@@ -192,18 +176,11 @@ func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error
 
 	if isExtIgnored(urlParsed.Path) {
 		logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path)
-		return "", nil
 	}
 
-	if entry.Request.Method == "OPTIONS" {
-		logger.Log.Debugf("Dropped traffic entry due to its method: %s %s", entry.Request.Method, urlParsed.Path)
-		return "", nil
-	}
-
-	ctype := getRespCtype(&entry.Response)
+	ctype := getRespCtype(entry.Response)
 	if isCtypeIgnored(ctype) {
 		logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype)
-		return "", nil
 	}
 
 	if entry.Response.Status < 100 {
@@ -216,19 +193,9 @@ func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error
 		return "", nil
 	}
 
-	if entry.Response.Status == 502 || entry.Response.Status == 503 || entry.Response.Status == 504 {
-		logger.Log.Debugf("Dropped traffic entry due to temporary server error: %s", entry.StartedDateTime)
-		return "", nil
-	}
-
-	var split []string
-	if urlParsed.RawPath != "" {
-		split = strings.Split(urlParsed.RawPath, "/")
-	} else {
-		split = strings.Split(urlParsed.Path, "/")
-	}
-	node := g.tree.getOrSet(split, new(openapi.PathObj), entryWithSource.Id)
-	opObj, err := handleOpObj(entryWithSource, node.pathObj)
+	split := strings.Split(urlParsed.Path, "/")
+	node := g.tree.getOrSet(split, new(openapi.PathObj))
+	opObj, err := handleOpObj(entry, node.ops)
 
 	if opObj != nil {
 		return opObj.OperationID, err
@@ -237,8 +204,7 @@ func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error
 	return "", err
 }
 
-func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj) (*openapi.Operation, error) {
-	entry := entryWithSource.Entry
+func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation, error) {
 	isSuccess := 100 <= entry.Response.Status && entry.Response.Status < 400
 	opObj, wasMissing, err := getOpObj(pathObj, entry.Request.Method, isSuccess)
 	if err != nil {
@@ -250,172 +216,73 @@ func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj) (*o
 		return nil, nil
 	}
 
-	err = handleRequest(&entry.Request, opObj, isSuccess, entryWithSource.Id)
+	err = handleRequest(entry.Request, opObj, isSuccess)
 	if err != nil {
 		return nil, err
 	}
 
-	err = handleResponse(&entry.Response, opObj, isSuccess, entryWithSource.Id)
+	err = handleResponse(entry.Response, opObj, isSuccess)
 	if err != nil {
 		return nil, err
 	}
 
-	err = handleCounters(opObj, isSuccess, entryWithSource)
-	if err != nil {
-		return nil, err
-	}
-
-	setSampleID(&opObj.Extensions, entryWithSource.Id)
-
 	return opObj, nil
 }
 
-func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *EntryWithSource) error {
-	// TODO: if performance around DecodeExtension+SetExtension is bad, store counters as separate maps
-	counter := Counter{}
-	counterMap := CounterMap{}
-	prevTs := 0.0
-	if opObj.Extensions == nil {
-		opObj.Extensions = openapi.Extensions{}
-	} else {
-		if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
-			err := opObj.Extensions.DecodeExtension(CountersTotal, &counter)
-			if err != nil {
-				return err
-			}
-		}
-
-		if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
-			err := opObj.Extensions.DecodeExtension(CountersPerSource, &counterMap)
-			if err != nil {
-				return err
-			}
-		}
-
-		if _, ok := opObj.Extensions.Extension(LastSeenTS); ok {
-			err := opObj.Extensions.DecodeExtension(LastSeenTS, &prevTs)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	var counterPerSource *Counter
-	if existing, ok := counterMap[entryWithSource.Source]; ok {
-		counterPerSource = existing
-	} else {
-		counterPerSource = new(Counter)
-		counterMap[entryWithSource.Source] = counterPerSource
-	}
-
-	started, err := datetime.Parse(entryWithSource.Entry.StartedDateTime, time.UTC)
-	if err != nil {
-		return err
-	}
-
-	ts := float64(started.UnixNano()) / float64(time.Millisecond) / 1000
-	rt := float64(entryWithSource.Entry.Time) / 1000
-
-	dur := 0.0
-	if prevTs != 0 && ts >= prevTs {
-		dur = ts - prevTs
-	}
-
-	counter.addEntry(ts, rt, success, dur)
-	counterPerSource.addEntry(ts, rt, success, dur)
-
-	err = opObj.Extensions.SetExtension(LastSeenTS, ts)
-	if err != nil {
-		return err
-	}
-
-	err = opObj.Extensions.SetExtension(CountersTotal, counter)
-	if err != nil {
-		return err
-	}
-
-	err = opObj.Extensions.SetExtension(CountersPerSource, counterMap)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool, sampleId string) error {
+func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) error {
 	// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
-	urlParsed, err := url.Parse(req.URL)
-	if err != nil {
-		return err
-	}
-
-	qs := make([]har.NVP, 0)
-	for name, vals := range urlParsed.Query() {
-		for _, val := range vals {
-			qs = append(qs, har.NVP{Name: name, Value: val})
-		}
-	}
-
-	if len(qs) != len(req.QueryString) {
-		logger.Log.Warningf("QStr params in HAR do not match URL: %s", req.URL)
-	}
 
 	qstrGW := nvParams{
-		In:             openapi.InQuery,
-		Pairs:          qs,
+		In: openapi.InQuery,
+		Pairs: func() []NVPair {
+			return qstrToNVP(req.QueryString)
+		},
 		IsIgnored:      func(name string) bool { return false },
 		GeneralizeName: func(name string) string { return name },
 	}
-	handleNameVals(qstrGW, &opObj.Parameters, false, sampleId)
+	handleNameVals(qstrGW, &opObj.Parameters)
 
 	hdrGW := nvParams{
-		In:             openapi.InHeader,
-		Pairs:          req.Headers,
+		In: openapi.InHeader,
+		Pairs: func() []NVPair {
+			return hdrToNVP(req.Headers)
+		},
 		IsIgnored:      isHeaderIgnored,
 		GeneralizeName: strings.ToLower,
 	}
-	handleNameVals(hdrGW, &opObj.Parameters, true, sampleId)
+	handleNameVals(hdrGW, &opObj.Parameters)
 
-	if isSuccess {
-		reqBody, err := getRequestBody(req, opObj)
+	if req.PostData != nil && req.PostData.Text != "" && isSuccess {
+		reqBody, err := getRequestBody(req, opObj, isSuccess)
 		if err != nil {
 			return err
 		}
 
 		if reqBody != nil {
-			setSampleID(&reqBody.Extensions, sampleId)
-
-			if req.PostData.Text == "" {
-				reqBody.Required = false
-			} else {
-
-				reqCtype, _ := getReqCtype(req)
-				reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype, sampleId)
-				if err != nil {
-					return err
-				}
-
-				_ = reqMedia
+			reqCtype := getReqCtype(req)
+			reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype, err)
+			if err != nil {
+				return err
 			}
+
+			_ = reqMedia
 		}
 	}
 	return nil
 }
 
-func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool, sampleId string) error {
+func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool) error {
 	// TODO: we don't support "default" response
 	respObj, err := getResponseObj(resp, opObj, isSuccess)
 	if err != nil {
 		return err
 	}
 
-	setSampleID(&respObj.Extensions, sampleId)
-
-	handleRespHeaders(resp.Headers, respObj, sampleId)
+	handleRespHeaders(resp.Headers, respObj)
 
 	respCtype := getRespCtype(resp)
 	respContent := respObj.Content
-	respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype, sampleId)
+	respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype, err)
 	if err != nil {
 		return err
 	}
@@ -423,7 +290,7 @@ func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool
 	return nil
 }
 
-func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj, sampleId string) {
+func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj) {
 	visited := map[string]*openapi.HeaderObj{}
 	for _, pair := range reqHeaders {
 		if isHeaderIgnored(pair.Name) {
@@ -445,8 +312,6 @@ func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj, sa
 			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
 		}
 		visited[nameGeneral] = param
-
-		setSampleID(&param.Extensions, sampleId)
 	}
 
 	// maintain "required" flag
@@ -465,159 +330,55 @@ func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj, sa
 			}
 		}
 	}
+
+	return
 }
 
-func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, sampleId string) (*openapi.MediaType, error) {
+func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err error) (*openapi.MediaType, error) {
 	content, found := respContent[ctype]
 	if !found {
 		respContent[ctype] = &openapi.MediaType{}
 		content = respContent[ctype]
 	}
 
-	setSampleID(&content.Extensions, sampleId)
-
 	var text string
-	var isBinary bool
 	if reqResp.Req != nil {
-		isBinary, _, text = reqResp.Req.PostData.B64Decoded()
+		text = reqResp.Req.PostData.Text
 	} else {
-		isBinary, _, text = reqResp.Resp.Content.B64Decoded()
+		text = decRespText(reqResp.Resp.Content)
 	}
 
-	if !isBinary && text != "" {
-		var exampleMsg []byte
-		// try treating it as json
-		anyVal, isJSON := anyJSON(text)
-		if isJSON {
-			// re-marshal with forced indent
-			if msg, err := json.MarshalIndent(anyVal, "", "\t"); err != nil {
-				panic("Failed to re-marshal value, super-strange")
-			} else {
-				exampleMsg = msg
-			}
-		} else {
-			if msg, err := json.Marshal(text); err != nil {
-				return nil, err
-			} else {
-				exampleMsg = msg
-			}
+	var exampleMsg []byte
+	// try treating it as json
+	any, isJSON := anyJSON(text)
+	if isJSON {
+		// re-marshal with forced indent
+		exampleMsg, err = json.MarshalIndent(any, "", "\t")
+		if err != nil {
+			panic("Failed to re-marshal value, super-strange")
 		}
-
-		if ctype == "application/x-www-form-urlencoded" && reqResp.Req != nil {
-			handleFormDataUrlencoded(text, content)
-		} else if strings.HasPrefix(ctype, "multipart/form-data") && reqResp.Req != nil {
-			_, params := getReqCtype(reqResp.Req)
-			handleFormDataMultipart(text, content, params)
-		}
-
-		if content.Example == nil && len(exampleMsg) > len(content.Example) {
-			content.Example = exampleMsg
+	} else {
+		exampleMsg, err = json.Marshal(text)
+		if err != nil {
+			return nil, err
 		}
 	}
 
+	content.Example = exampleMsg
 	return respContent[ctype], nil
 }
 
-func handleFormDataUrlencoded(text string, content *openapi.MediaType) {
-	formData, err := url.ParseQuery(text)
-	if err != nil {
-		logger.Log.Warningf("Could not decode urlencoded: %s", err)
-		return
-	}
-
-	parts := make([]PartWithBody, 0)
-	for name, vals := range formData {
-		for _, val := range vals {
-			part := new(multipart.Part)
-			part.Header = textproto.MIMEHeader{}
-			part.Header.Add("Content-Disposition", "form-data; name=\""+name+"\";")
-			parts = append(parts, PartWithBody{part: part, body: []byte(val)})
-		}
-	}
-	handleFormData(content, parts)
-}
-
-func handleFormData(content *openapi.MediaType, parts []PartWithBody) {
-	hadSchema := true
-	if content.Schema == nil {
-		hadSchema = false // will use it for required flags
-		content.Schema = new(openapi.SchemaObj)
-		content.Schema.Type = openapi.Types{openapi.TypeObject}
-		content.Schema.Properties = openapi.Schemas{}
-	}
-
-	props := &content.Schema.Properties
-	seenNames := map[string]struct{}{} // set equivalent in Go, yikes
-	for _, pwb := range parts {
-		name := pwb.part.FormName()
-		seenNames[name] = struct{}{}
-		existing, found := (*props)[name]
-		if !found {
-			existing = new(openapi.SchemaObj)
-			existing.Type = openapi.Types{openapi.TypeString}
-			(*props)[name] = existing
-
-			ctype := pwb.part.Header.Get("content-type")
-			if ctype != "" {
-				if existing.Keywords == nil {
-					existing.Keywords = map[string]json.RawMessage{}
-				}
-				existing.Keywords["contentMediaType"], _ = json.Marshal(ctype)
-			}
-		}
-
-		addSchemaExample(existing, string(pwb.body))
-	}
-
-	// handle required flag
-	if content.Schema.Required == nil {
-		if !hadSchema {
-			content.Schema.Required = make([]string, 0)
-			for name := range seenNames {
-				content.Schema.Required = append(content.Schema.Required, name)
-			}
-			sort.Strings(content.Schema.Required)
-		} // else it's a known schema with no required fields
-	} else {
-		content.Schema.Required = intersectSliceWithMap(content.Schema.Required, seenNames)
-		sort.Strings(content.Schema.Required)
-	}
-}
-
-type PartWithBody struct {
-	part *multipart.Part
-	body []byte
-}
-
-func handleFormDataMultipart(text string, content *openapi.MediaType, ctypeParams map[string]string) {
-	boundary, ok := ctypeParams["boundary"]
-	if !ok {
-		logger.Log.Errorf("Multipart header has no boundary")
-		return
-	}
-	mpr := multipart.NewReader(strings.NewReader(text), boundary)
-
-	parts := make([]PartWithBody, 0)
-	for {
-		part, err := mpr.NextPart()
-		if err == io.EOF {
-			break
-		}
+func decRespText(content *har.Content) (res string) {
+	res = string(content.Text)
+	if content.Encoding == "base64" {
+		data, err := base64.StdEncoding.DecodeString(res)
 		if err != nil {
-			logger.Log.Errorf("Cannot parse multipart body: %v", err)
-			break
+			logger.Log.Warningf("error decoding response text as base64: %s", err)
+		} else {
+			res = string(data)
 		}
-		defer part.Close()
-
-		body, err := ioutil.ReadAll(part)
-		if err != nil {
-			logger.Log.Errorf("Error reading multipart Part %s: %v", part.Header, err)
-		}
-
-		parts = append(parts, PartWithBody{part: part, body: body})
 	}
-
-	handleFormData(content, parts)
+	return
 }
 
 func getRespCtype(resp *har.Response) string {
@@ -636,7 +397,8 @@ func getRespCtype(resp *har.Response) string {
 	return mediaType
 }
 
-func getReqCtype(req *har.Request) (ctype string, params map[string]string) {
+func getReqCtype(req *har.Request) string {
+	var ctype string
 	ctype = req.PostData.MimeType
 	for _, hdr := range req.Headers {
 		if strings.ToLower(hdr.Name) == "content-type" {
@@ -644,16 +406,11 @@ func getReqCtype(req *har.Request) (ctype string, params map[string]string) {
 		}
 	}
 
-	if ctype == "" {
-		return "", map[string]string{}
-	}
-
-	mediaType, params, err := mime.ParseMediaType(ctype)
+	mediaType, _, err := mime.ParseMediaType(ctype)
 	if err != nil {
-		logger.Log.Errorf("Cannot parse Content-Type header %q: %v", ctype, err)
-		return "", map[string]string{}
+		return ""
 	}
-	return mediaType, params
+	return mediaType
 }
 
 func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool) (*openapi.ResponseObj, error) {
@@ -683,14 +440,9 @@ func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool
 	return resResponse, nil
 }
 
-func getRequestBody(req *har.Request, opObj *openapi.Operation) (*openapi.RequestBodyObj, error) {
+func getRequestBody(req *har.Request, opObj *openapi.Operation, isSuccess bool) (*openapi.RequestBodyObj, error) {
 	if opObj.RequestBody == nil {
-		// create if there is body in request
-		if req.PostData.Text != "" {
-			opObj.RequestBody = &openapi.RequestBodyObj{Description: "Generic request body", Required: true, Content: map[string]*openapi.MediaType{}}
-		} else {
-			return nil, nil
-		}
+		opObj.RequestBody = &openapi.RequestBodyObj{Description: "Generic request body", Required: true, Content: map[string]*openapi.MediaType{}}
 	}
 
 	reqBody, err := opObj.RequestBody.ResolveRequestBody(reqBodyResolver)
@@ -698,6 +450,9 @@ func getRequestBody(req *har.Request, opObj *openapi.Operation) (*openapi.Reques
 		return nil, err
 	}
 
+	// TODO: maintain required flag for it, but only consider successful responses
+	//reqBody.Content[]
+
 	return reqBody, nil
 }
 

agent/pkg/oas/specgen_test.go

@@ -1,95 +1,53 @@
 package oas
 
 import (
-	"bytes"
 	"encoding/json"
-	"io/ioutil"
-	"net"
-	"os"
-	"regexp"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
 	"github.com/chanced/openapi"
+	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
-	"github.com/wI2L/jsondiff"
-
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/agent/pkg/har"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
 )
 
-func GetFakeDBConn(send string) *basenine.Connection {
-	dummyConn := new(basenine.Connection)
-	dummyConn.Conn = FakeConn{
-		sendBuffer:    bytes.NewBufferString(send),
-		receiveBuffer: bytes.NewBufferString(""),
-	}
-	return dummyConn
-}
-
 // if started via env, write file into subdir
-func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) string {
-	content, err := json.MarshalIndent(spec, "", "  ")
-	if err != nil {
-		panic(err)
-	}
-
+func writeFiles(label string, spec *openapi.OpenAPI) {
 	if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
 		path := "./oas-samples"
 		err := os.MkdirAll(path, 0o755)
 		if err != nil {
 			panic(err)
 		}
+		content, err := json.MarshalIndent(spec, "", "\t")
+		if err != nil {
+			panic(err)
+		}
 		err = ioutil.WriteFile(path+"/"+label+".json", content, 0644)
 		if err != nil {
 			panic(err)
 		}
-		t.Logf("Written: %s", label)
-	} else {
-		t.Logf("%s", string(content))
 	}
-	return string(content)
 }
 
 func TestEntries(t *testing.T) {
-	//logger.InitLoggerStd(logging.INFO) causes race condition
 	files, err := getFiles("./test_artifacts/")
+	// files, err = getFiles("/media/bigdisk/UP9")
 	if err != nil {
 		t.Log(err)
 		t.FailNow()
 	}
+	GetOasGeneratorInstance().Start()
 
-	gen := NewDefaultOasGenerator()
-	gen.serviceSpecs = new(sync.Map)
-	loadStartingOAS("test_artifacts/catalogue.json", "catalogue", gen.serviceSpecs)
-	loadStartingOAS("test_artifacts/trcc.json", "trcc-api-service", gen.serviceSpecs)
-
-	go func() {
-		for {
-			time.Sleep(1 * time.Second)
-			gen.serviceSpecs.Range(func(key, val interface{}) bool {
-				svc := key.(string)
-				t.Logf("Getting spec for %s", svc)
-				gen := val.(*SpecGen)
-				_, err := gen.GetSpec()
-				if err != nil {
-					t.Error(err)
-				}
-				return true
-			})
-		}
-	}()
-
-	cnt, err := feedEntries(files, true, gen)
-	if err != nil {
+	if err := feedEntries(files); err != nil {
 		t.Log(err)
 		t.Fail()
 	}
 
+	loadStartingOAS()
+
 	svcs := strings.Builder{}
-	gen.serviceSpecs.Range(func(key, val interface{}) bool {
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
 		gen := val.(*SpecGen)
 		svc := key.(string)
 		svcs.WriteString(svc + ",")
@@ -111,7 +69,7 @@ func TestEntries(t *testing.T) {
 		return true
 	})
 
-	gen.serviceSpecs.Range(func(key, val interface{}) bool {
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
 		svc := key.(string)
 		gen := val.(*SpecGen)
 		spec, err := gen.GetSpec()
@@ -120,34 +78,33 @@ func TestEntries(t *testing.T) {
 			t.FailNow()
 		}
 
-		outputSpec(svc, spec, t)
+		specText, _ := json.MarshalIndent(spec, "", "\t")
+		t.Logf("%s", string(specText))
 
 		err = spec.Validate()
 		if err != nil {
 			t.Log(err)
 			t.FailNow()
 		}
+		writeFiles(svc, spec)
 
 		return true
 	})
 
-	logger.Log.Infof("Total entries: %d", cnt)
 }
 
-func TestFileSingle(t *testing.T) {
-	gen := NewDefaultOasGenerator()
-	gen.serviceSpecs = new(sync.Map)
-	// loadStartingOAS()
-	file := "test_artifacts/params.har"
-	files := []string{file}
-	cnt, err := feedEntries(files, true, gen)
+func TestFileLDJSON(t *testing.T) {
+	GetOasGeneratorInstance().Start()
+	file := "test_artifacts/output_rdwtyeoyrj.har.ldjson"
+	err := feedFromLDJSON(file)
 	if err != nil {
 		logger.Log.Warning("Failed processing file: " + err.Error())
 		t.Fail()
 	}
 
-	gen.serviceSpecs.Range(func(key, val interface{}) bool {
-		svc := key.(string)
+	loadStartingOAS()
+
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(_, val interface{}) bool {
 		gen := val.(*SpecGen)
 		spec, err := gen.GetSpec()
 		if err != nil {
@@ -155,7 +112,8 @@ func TestFileSingle(t *testing.T) {
 			t.FailNow()
 		}
 
-		specText := outputSpec(svc, spec, t)
+		specText, _ := json.MarshalIndent(spec, "", "\t")
+		t.Logf("%s", string(specText))
 
 		err = spec.Validate()
 		if err != nil {
@@ -163,43 +121,12 @@ func TestFileSingle(t *testing.T) {
 			t.FailNow()
 		}
 
-		expected, err := ioutil.ReadFile(file + ".spec.json")
-		if err != nil {
-			t.Errorf(err.Error())
-			t.FailNow()
-		}
-
-		patFloatPrecision := regexp.MustCompile(`(\d+\.\d{1,2})(\d*)`)
-
-		expected = []byte(patUuid4.ReplaceAllString(string(expected), "<UUID4>"))
-		specText = patUuid4.ReplaceAllString(specText, "<UUID4>")
-		expected = []byte(patFloatPrecision.ReplaceAllString(string(expected), "$1"))
-		specText = patFloatPrecision.ReplaceAllString(specText, "$1")
-
-		diff, err := jsondiff.CompareJSON(expected, []byte(specText))
-		if err != nil {
-			t.Errorf(err.Error())
-			t.FailNow()
-		}
-
-		if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
-			err = ioutil.WriteFile(file+".spec.json", []byte(specText), 0644)
-			if err != nil {
-				panic(err)
-			}
-		}
-
-		if len(diff) > 0 {
-			t.Errorf("Generated spec does not match expected:\n%s", diff.String())
-		}
-
 		return true
 	})
-
-	logger.Log.Infof("Processed entries: %d", cnt)
 }
 
-func loadStartingOAS(file string, label string, specs *sync.Map) {
+func loadStartingOAS() {
+	file := "test_artifacts/catalogue.json"
 	fd, err := os.Open(file)
 	if err != nil {
 		panic(err)
@@ -218,41 +145,30 @@ func loadStartingOAS(file string, label string, specs *sync.Map) {
 		panic(err)
 	}
 
-	gen := NewGen(label)
+	gen := NewGen("catalogue")
 	gen.StartFromSpec(doc)
 
-	specs.Store(label, gen)
+	GetOasGeneratorInstance().ServiceSpecs.Store("catalogue", gen)
+
+	return
 }
 
 func TestEntriesNegative(t *testing.T) {
-	gen := NewDefaultOasGenerator()
-	gen.serviceSpecs = new(sync.Map)
 	files := []string{"invalid"}
-	_, err := feedEntries(files, false, gen)
+	err := feedEntries(files)
 	if err == nil {
 		t.Logf("Should have failed")
 		t.Fail()
 	}
 }
 
-func TestEntriesPositive(t *testing.T) {
-	gen := NewDefaultOasGenerator()
-	gen.serviceSpecs = new(sync.Map)
-	files := []string{"test_artifacts/params.har"}
-	_, err := feedEntries(files, false, gen)
-	if err != nil {
-		t.Logf("Failed")
-		t.Fail()
-	}
-}
-
 func TestLoadValidHAR(t *testing.T) {
 	inp := `{"startedDateTime": "2021-02-03T07:48:12.959000+00:00", "time": 1, "request": {"method": "GET", "url": "http://unresolved_target/1.0.0/health", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "queryString": [], "headersSize": -1, "bodySize": -1}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "content": {"size": 2, "mimeType": "", "text": "OK"}, "redirectURL": "", "headersSize": -1, "bodySize": 2}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 1}}`
 	var entry *har.Entry
 	var err = json.Unmarshal([]byte(inp), &entry)
 	if err != nil {
 		t.Logf("Failed to decode entry: %s", err)
-		t.FailNow() // demonstrates the problem of `martian` HAR library
+		// t.FailNow() demonstrates the problem of library
 	}
 }
 
@@ -277,18 +193,5 @@ func TestLoadValid3_1(t *testing.T) {
 		t.Log(err)
 		t.FailNow()
 	}
+	return
 }
-
-type FakeConn struct {
-	sendBuffer    *bytes.Buffer
-	receiveBuffer *bytes.Buffer
-}
-
-func (f FakeConn) Read(p []byte) (int, error)       { return f.sendBuffer.Read(p) }
-func (f FakeConn) Write(p []byte) (int, error)      { return f.receiveBuffer.Write(p) }
-func (FakeConn) Close() error                       { return nil }
-func (FakeConn) LocalAddr() net.Addr                { return nil }
-func (FakeConn) RemoteAddr() net.Addr               { return nil }
-func (FakeConn) SetDeadline(t time.Time) error      { return nil }
-func (FakeConn) SetReadDeadline(t time.Time) error  { return nil }
-func (FakeConn) SetWriteDeadline(t time.Time) error { return nil }

agent/pkg/oas/test_artifacts/output_rdwtyeoyrj.har.ldjson

@@ -1,4 +1,4 @@
-{"messageType": "http", "_source": "some-source", ",firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
+{"messageType": "http", "_source": null, "firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
 {"_id": "", "startedDateTime": "2021-07-26T11:14:17.78415179Z", "time": 13, "request": {"method": "GET", "url": "http://catalogue/catalogue/size?tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "x-some", "value": "demo val"},{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [{"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "22"}], "content": {"size": 22, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInNpemUiOjl9", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 22}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 13}}
 {"_id": "", "startedDateTime": "2021-07-26T11:14:17.784918698Z", "time": 19, "request": {"method": "GET", "url": "http://catalogue/catalogue?page=1&size=6&tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "page", "value": "1"}, {"name": "size", "value": "6"}, {"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "1927"}], "content": {"size": 1927, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19LHsiaWQiOiI4MzdhYjE0MS0zOTllLTRjMWYtOWFiYy1iYWNlNDAyOTZiYWMiLCJuYW1lIjoiQ2F0IHNvY2tzIiwiZGVzY3JpcHRpb24iOiJjb25zZXF1YXQgYW1ldCBjdXBpZGF0YXQgbWluaW0gbGFib3J1bSB0ZW1wb3IgZWxpdCBleCBjb25zZXF1YXQgaW4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2NrczIuanBnIl0sInByaWNlIjoxNSwiY291bnQiOjE3NSwidGFnIjpbImJyb3duIiwiZm9ybWFsIiwiZ3JlZW4iXX1dCg==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1927}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 19}}
 {"_id": "", "startedDateTime": "2021-07-26T11:14:17.78418182Z", "time": 7, "request": {"method": "GET", "url": "http://catalogue/tags", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "107"}], "content": {"size": 107, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInRhZ3MiOlsiYnJvd24iLCJnZWVrIiwiZm9ybWFsIiwiYmx1ZSIsInNraW4iLCJyZWQiLCJhY3Rpb24iLCJzcG9ydCIsImJsYWNrIiwibWFnaWMiLCJncmVlbiJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 107}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 7}}

agent/pkg/oas/test_artifacts/output_ysuwqrdktj.har

@@ -13,14 +13,10 @@
         "time": 1022,
         "request": {
           "method": "GET",
-          "url": "http://trcc-api-service/proxies;matrixparam=example/",
+          "url": "http://trcc-api-service/proxies/",
           "httpVersion": "",
           "cookies": [],
           "headers": [
-            {
-              "name": "X-Custom-Demo-Header",
-              "value": "demo"
-            },
             {
               "name": "Sec-Fetch-Dest",
               "value": "empty"
@@ -128,10 +124,6 @@
           "httpVersion": "",
           "cookies": [],
           "headers": [
-            {
-              "name": "X-Custom-Demo-Header2",
-              "value": "demo2"
-            },
             {
               "name": "Vary",
               "value": "Origin"
@@ -24576,7 +24568,7 @@
           "bodySize": -1
         },
         "response": {
-          "status": 308,
+          "status": 200,
           "statusText": "OK",
           "httpVersion": "",
           "cookies": [],
@@ -24643,7 +24635,7 @@
         "time": 1,
         "request": {
           "method": "GET",
-          "url": "http://trcc-api-service/models/aws_kong5/suites/all/runs.png",
+          "url": "http://trcc-api-service/models/aws_kong5/suites/all/runs",
           "httpVersion": "",
           "cookies": [],
           "headers": [
@@ -24902,7 +24894,7 @@
           "bodySize": -1
         },
         "response": {
-          "status": 0,
+          "status": 200,
           "statusText": "OK",
           "httpVersion": "",
           "cookies": [],

agent/pkg/oas/test_artifacts/params.har

@@ -1,790 +0,0 @@
-{
-  "log": {
-    "version": "1.2",
-    "creator": {
-      "name": "mitmproxy har_dump",
-      "version": "0.1",
-      "comment": "mitmproxy version mitmproxy 4.0.4"
-    },
-    "entries": [
-      {
-        "startedDateTime": "2019-09-06T06:14:43.864529+00:00",
-        "time": 111,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/e21f7112-3d3b-4632-9da3-a4af2e0e9166/sub1",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "headersSize": 1542,
-          "bodySize": 0,
-          "queryString": []
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "mimeType": "text/html",
-            "text": "",
-            "size": 0
-          },
-          "redirectURL": "",
-          "headersSize": 245,
-          "bodySize": 39
-        },
-        "cache": {},
-        "timings": {
-          "send": 22,
-          "receive": 2,
-          "wait": 87,
-          "connect": -1,
-          "ssl": -1
-        },
-        "serverIPAddress": "54.210.29.33"
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:18.747122+00:00",
-        "time": 630,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a/sub2",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "queryString": [],
-          "headersSize": 1542,
-          "bodySize": 0
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "content": {
-            "size": 39,
-            "compression": -20,
-            "mimeType": "application/json",
-            "text": "null"
-          },
-          "redirectURL": "",
-          "headersSize": 248,
-          "bodySize": 39
-        },
-        "cache": {},
-        "timings": {
-          "send": 14,
-          "receive": 4,
-          "wait": 350,
-          "connect": 262,
-          "ssl": -1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:19.747122+00:00",
-        "time": 630,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a;mparam=matrixparam",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "queryString": [],
-          "headersSize": 1542,
-          "bodySize": 0
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "content": {
-            "size": 39,
-            "compression": -20,
-            "mimeType": "application/json",
-            "text": "null"
-          },
-          "redirectURL": "",
-          "headersSize": 248,
-          "bodySize": 39
-        },
-        "cache": {},
-        "timings": {
-          "send": 14,
-          "receive": 4,
-          "wait": 350,
-          "connect": 262,
-          "ssl": -1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:20.047122+00:00",
-        "time": 630,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/appears-once",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "queryString": [],
-          "headersSize": 1542,
-          "bodySize": 0
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "content": {
-            "size": 39,
-            "compression": -20,
-            "mimeType": "application/json",
-            "text": "null"
-          },
-          "redirectURL": "",
-          "headersSize": 248,
-          "bodySize": 39
-        },
-        "cache": {},
-        "timings": {
-          "send": 14,
-          "receive": 4,
-          "wait": 350,
-          "connect": 262,
-          "ssl": -1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:20.747122+00:00",
-        "time": 630,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/appears-twice",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "queryString": [],
-          "headersSize": 1542,
-          "bodySize": 0
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "content": {
-            "size": 39,
-            "compression": -20,
-            "mimeType": "application/json",
-            "text": "null"
-          },
-          "redirectURL": "",
-          "headersSize": 248,
-          "bodySize": 39
-        },
-        "cache": {},
-        "timings": {
-          "send": 14,
-          "receive": 4,
-          "wait": 350,
-          "connect": 262,
-          "ssl": -1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:21.747122+00:00",
-        "time": 630,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/appears-twice",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "queryString": [],
-          "headersSize": 1542,
-          "bodySize": 0
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "content": {
-            "size": 39,
-            "compression": -20,
-            "mimeType": "application/json",
-            "text": "null"
-          },
-          "redirectURL": "",
-          "headersSize": 248,
-          "bodySize": 39
-        },
-        "cache": {},
-        "timings": {
-          "send": 14,
-          "receive": 4,
-          "wait": 350,
-          "connect": 262,
-          "ssl": -1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:20.747122+00:00",
-        "time": 1,
-        "request": {
-          "method": "POST",
-          "url": "https://httpbin.org/form-urlencoded",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-            {
-              "name": "Content-Type",
-              "value": "application/x-www-form-urlencoded"
-            }
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": "agent-id=ade&callback-url=&token=sometoken"
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:21.747122+00:00",
-        "time": 1,
-        "request": {
-          "method": "POST",
-          "url": "https://httpbin.org/form-urlencoded",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-            {
-              "name": "Content-Type",
-              "value": "application/x-www-form-urlencoded"
-            }
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": "agent-id=ade&callback-url=&token=sometoken-second-val&optional=another"
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:22.747122+00:00",
-        "time": 1,
-        "request": {
-          "method": "POST",
-          "url": "https://httpbin.org/form-multipart",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-            {
-              "name": "Content-Type",
-              "value": "multipart/form-data; boundary=BOUNDARY"
-            }
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n"
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 62,
-            "mimeType": "",
-            "text": "{}"
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 62
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:21.757122+00:00",
-        "time": 1,
-        "request": {
-          "method": "POST",
-          "url": "https://httpbin.org/body-optional",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": ""
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:21.747122+00:00",
-        "time": 1,
-        "request": {
-          "method": "POST",
-          "url": "https://httpbin.org/body-optional",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-            {
-              "name": "Content-Type",
-              "value": "application/json"
-            }
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": "{\"key\", \"val\"}"
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:21.757122+00:00",
-        "time": 1,
-        "request": {
-          "method": "POST",
-          "url": "https://httpbin.org/body-optional",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": ""
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:21.757122+00:00",
-        "time": 1,
-        "request": {
-          "method": "POST",
-          "url": "https://httpbin.org/body-required",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": "body exists"
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:22.000000+00:00",
-        "time": 1,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/param-patterns/prefix-gibberish-fine/234324",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": ""
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:22.000001+00:00",
-        "time": 1,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/param-patterns/prefix-gibberish-sfdlasdfkadf87sd93284q24r/1",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": ""
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-     {
-        "startedDateTime": "2019-09-06T06:16:22.000002+00:00",
-        "time": 1,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/param-patterns/prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf/static",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": ""
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-     {
-        "startedDateTime": "2019-09-06T06:16:22.000003+00:00",
-        "time": 1,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/param-patterns/prefix-gibberish-4jk5l2345h2452l4352435jlk45",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": ""
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-     {
-        "startedDateTime": "2019-09-06T06:16:22.000004+00:00",
-        "time": 1,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/param-patterns/prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": ""
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:22.000002+00:00",
-        "time": 1,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/param-patterns/prefix-gibberish-afterwards/23421",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": ""
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      }
-    ]
-  }
-}

agent/pkg/oas/test_artifacts/params.har.spec.json

@@ -1,897 +0,0 @@
-{
-  "openapi": "3.1.0",
-  "info": {
-    "title": "https://httpbin.org",
-    "description": "Mizu observed 19 entries (0 failed), at 0.10 hits/s, average response time is 0.17 seconds",
-    "version": "1.0"
-  },
-  "servers": [
-    {
-      "url": "https://httpbin.org"
-    }
-  ],
-  "paths": {
-    "/appears-once": {
-      "get": {
-        "summary": "/appears-once",
-        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds",
-        "operationId": "<UUID4>",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": null,
-                "x-sample-entry": "000000000000000000000004"
-              }
-            },
-            "x-sample-entry": "000000000000000000000004"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750580.04,
-            "lastSeen": 1567750580.04,
-            "sumRT": 0.63,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750580.04,
-          "lastSeen": 1567750580.04,
-          "sumRT": 0.63,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750580.04,
-        "x-sample-entry": "000000000000000000000004"
-      }
-    },
-    "/appears-twice": {
-      "get": {
-        "summary": "/appears-twice",
-        "description": "Mizu observed 2 entries (0 failed), at 0.50 hits/s, average response time is 0.63 seconds",
-        "operationId": "<UUID4>",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": null,
-                "x-sample-entry": "000000000000000000000006"
-              }
-            },
-            "x-sample-entry": "000000000000000000000006"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 2,
-            "failures": 0,
-            "firstSeen": 1567750580.74,
-            "lastSeen": 1567750581.74,
-            "sumRT": 1.26,
-            "sumDuration": 1
-          }
-        },
-        "x-counters-total": {
-          "entries": 2,
-          "failures": 0,
-          "firstSeen": 1567750580.74,
-          "lastSeen": 1567750581.74,
-          "sumRT": 1.26,
-          "sumDuration": 1
-        },
-        "x-last-seen-ts": 1567750581.74,
-        "x-sample-entry": "000000000000000000000006"
-      }
-    },
-    "/body-optional": {
-      "post": {
-        "summary": "/body-optional",
-        "description": "Mizu observed 3 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
-        "operationId": "<UUID4>",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000012"
-              }
-            },
-            "x-sample-entry": "000000000000000000000012"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 3,
-            "failures": 0,
-            "firstSeen": 1567750581.74,
-            "lastSeen": 1567750581.75,
-            "sumRT": 0.00,
-            "sumDuration": 0.01
-          }
-        },
-        "x-counters-total": {
-          "entries": 3,
-          "failures": 0,
-          "firstSeen": 1567750581.74,
-          "lastSeen": 1567750581.75,
-          "sumRT": 0.00,
-          "sumDuration": 0.01
-        },
-        "x-last-seen-ts": 1567750581.75,
-        "x-sample-entry": "000000000000000000000012",
-        "requestBody": {
-          "description": "Generic request body",
-          "content": {
-            "application/json": {
-              "example": "{\"key\", \"val\"}",
-              "x-sample-entry": "000000000000000000000011"
-            }
-          },
-          "x-sample-entry": "000000000000000000000012"
-        }
-      }
-    },
-    "/body-required": {
-      "post": {
-        "summary": "/body-required",
-        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
-        "operationId": "<UUID4>",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000013"
-              }
-            },
-            "x-sample-entry": "000000000000000000000013"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750581.75,
-            "lastSeen": 1567750581.75,
-            "sumRT": 0.00,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750581.75,
-          "lastSeen": 1567750581.75,
-          "sumRT": 0.00,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750581.75,
-        "x-sample-entry": "000000000000000000000013",
-        "requestBody": {
-          "description": "Generic request body",
-          "content": {
-            "": {
-              "example": "body exists",
-              "x-sample-entry": "000000000000000000000013"
-            }
-          },
-          "required": true,
-          "x-sample-entry": "000000000000000000000013"
-        }
-      }
-    },
-    "/form-multipart": {
-      "post": {
-        "summary": "/form-multipart",
-        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
-        "operationId": "<UUID4>",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "example": {},
-                "x-sample-entry": "000000000000000000000009"
-              }
-            },
-            "x-sample-entry": "000000000000000000000009"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750582.74,
-            "lastSeen": 1567750582.74,
-            "sumRT": 0.00,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750582.74,
-          "lastSeen": 1567750582.74,
-          "sumRT": 0.00,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750582.74,
-        "x-sample-entry": "000000000000000000000009",
-        "requestBody": {
-          "description": "Generic request body",
-          "content": {
-            "multipart/form-data": {
-              "schema": {
-                "type": "object",
-                "required": [
-                  "file",
-                  "path"
-                ],
-                "properties": {
-                  "file": {
-                    "type": "string",
-                    "contentMediaType": "application/json",
-                    "examples": [
-                      "{\"functions\": 123}"
-                    ]
-                  },
-                  "path": {
-                    "type": "string",
-                    "examples": [
-                      "/content/components"
-                    ]
-                  }
-                }
-              },
-              "example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n",
-              "x-sample-entry": "000000000000000000000009"
-            }
-          },
-          "required": true,
-          "x-sample-entry": "000000000000000000000009"
-        }
-      }
-    },
-    "/form-urlencoded": {
-      "post": {
-        "summary": "/form-urlencoded",
-        "description": "Mizu observed 2 entries (0 failed), at 0.50 hits/s, average response time is 0.00 seconds",
-        "operationId": "<UUID4>",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000008"
-              }
-            },
-            "x-sample-entry": "000000000000000000000008"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 2,
-            "failures": 0,
-            "firstSeen": 1567750580.74,
-            "lastSeen": 1567750581.74,
-            "sumRT": 0.00,
-            "sumDuration": 1
-          }
-        },
-        "x-counters-total": {
-          "entries": 2,
-          "failures": 0,
-          "firstSeen": 1567750580.74,
-          "lastSeen": 1567750581.74,
-          "sumRT": 0.00,
-          "sumDuration": 1
-        },
-        "x-last-seen-ts": 1567750581.74,
-        "x-sample-entry": "000000000000000000000008",
-        "requestBody": {
-          "description": "Generic request body",
-          "content": {
-            "application/x-www-form-urlencoded": {
-              "schema": {
-                "type": "object",
-                "required": [
-                  "agent-id",
-                  "callback-url",
-                  "token"
-                ],
-                "properties": {
-                  "agent-id": {
-                    "type": "string",
-                    "examples": [
-                      "ade"
-                    ]
-                  },
-                  "callback-url": {
-                    "type": "string",
-                    "examples": [
-                      ""
-                    ]
-                  },
-                  "optional": {
-                    "type": "string",
-                    "examples": [
-                      "another"
-                    ]
-                  },
-                  "token": {
-                    "type": "string",
-                    "examples": [
-                      "sometoken",
-                      "sometoken-second-val"
-                    ]
-                  }
-                }
-              },
-              "example": "agent-id=ade\u0026callback-url=\u0026token=sometoken",
-              "x-sample-entry": "000000000000000000000008"
-            }
-          },
-          "required": true,
-          "x-sample-entry": "000000000000000000000008"
-        }
-      }
-    },
-    "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}": {
-      "get": {
-        "tags": [
-          "param-patterns"
-        ],
-        "summary": "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}",
-        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
-        "operationId": "<UUID4>",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000014"
-              }
-            },
-            "x-sample-entry": "000000000000000000000014"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750582,
-            "lastSeen": 1567750582,
-            "sumRT": 0.00,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750582,
-          "lastSeen": 1567750582,
-          "sumRT": 0.00,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750582,
-        "x-sample-entry": "000000000000000000000014"
-      },
-      "parameters": [
-        {
-          "name": "prefixgibberishfineId",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "examples": {
-            "example #0": {
-              "value": "234324"
-            }
-          },
-          "x-sample-entry": "000000000000000000000014"
-        }
-      ]
-    },
-    "/param-patterns/{parampatternId}": {
-      "get": {
-        "tags": [
-          "param-patterns"
-        ],
-        "summary": "/param-patterns/{parampatternId}",
-        "description": "Mizu observed 2 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
-        "operationId": "<UUID4>",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000018"
-              }
-            },
-            "x-sample-entry": "000000000000000000000018"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 2,
-            "failures": 0,
-            "firstSeen": 1567750582.00,
-            "lastSeen": 1567750582.00,
-            "sumRT": 0.00,
-            "sumDuration": 9.53e-7
-          }
-        },
-        "x-counters-total": {
-          "entries": 2,
-          "failures": 0,
-          "firstSeen": 1567750582.00,
-          "lastSeen": 1567750582.00,
-          "sumRT": 0.00,
-          "sumDuration": 9.53e-7
-        },
-        "x-last-seen-ts": 1567750582.00,
-        "x-sample-entry": "000000000000000000000018"
-      },
-      "parameters": [
-        {
-          "name": "parampatternId",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string",
-            "pattern": "^prefix-gibberish-.+"
-          },
-          "examples": {
-            "example #0": {
-              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
-            },
-            "example #1": {
-              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
-            },
-            "example #2": {
-              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
-            },
-            "example #3": {
-              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
-            },
-            "example #4": {
-              "value": "prefix-gibberish-afterwards"
-            }
-          },
-          "x-sample-entry": "000000000000000000000019"
-        }
-      ]
-    },
-    "/param-patterns/{parampatternId}/1": {
-      "get": {
-        "tags": [
-          "param-patterns"
-        ],
-        "summary": "/param-patterns/{parampatternId}/1",
-        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
-        "operationId": "<UUID4>",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000015"
-              }
-            },
-            "x-sample-entry": "000000000000000000000015"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750582.00,
-            "lastSeen": 1567750582.00,
-            "sumRT": 0.00,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750582.00,
-          "lastSeen": 1567750582.00,
-          "sumRT": 0.00,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750582.00,
-        "x-sample-entry": "000000000000000000000015"
-      },
-      "parameters": [
-        {
-          "name": "parampatternId",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string",
-            "pattern": "^prefix-gibberish-.+"
-          },
-          "examples": {
-            "example #0": {
-              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
-            },
-            "example #1": {
-              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
-            },
-            "example #2": {
-              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
-            },
-            "example #3": {
-              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
-            },
-            "example #4": {
-              "value": "prefix-gibberish-afterwards"
-            }
-          },
-          "x-sample-entry": "000000000000000000000019"
-        }
-      ]
-    },
-    "/param-patterns/{parampatternId}/static": {
-      "get": {
-        "tags": [
-          "param-patterns"
-        ],
-        "summary": "/param-patterns/{parampatternId}/static",
-        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
-        "operationId": "<UUID4>",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000016"
-              }
-            },
-            "x-sample-entry": "000000000000000000000016"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750582.00,
-            "lastSeen": 1567750582.00,
-            "sumRT": 0.00,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750582.00,
-          "lastSeen": 1567750582.00,
-          "sumRT": 0.00,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750582.00,
-        "x-sample-entry": "000000000000000000000016"
-      },
-      "parameters": [
-        {
-          "name": "parampatternId",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string",
-            "pattern": "^prefix-gibberish-.+"
-          },
-          "examples": {
-            "example #0": {
-              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
-            },
-            "example #1": {
-              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
-            },
-            "example #2": {
-              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
-            },
-            "example #3": {
-              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
-            },
-            "example #4": {
-              "value": "prefix-gibberish-afterwards"
-            }
-          },
-          "x-sample-entry": "000000000000000000000019"
-        }
-      ]
-    },
-    "/param-patterns/{parampatternId}/{param1}": {
-      "get": {
-        "tags": [
-          "param-patterns"
-        ],
-        "summary": "/param-patterns/{parampatternId}/{param1}",
-        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
-        "operationId": "<UUID4>",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "x-sample-entry": "000000000000000000000019"
-              }
-            },
-            "x-sample-entry": "000000000000000000000019"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750582.00,
-            "lastSeen": 1567750582.00,
-            "sumRT": 0.00,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750582.00,
-          "lastSeen": 1567750582.00,
-          "sumRT": 0.00,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750582.00,
-        "x-sample-entry": "000000000000000000000019"
-      },
-      "parameters": [
-        {
-          "name": "param1",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "examples": {
-            "example #0": {
-              "value": "23421"
-            }
-          },
-          "x-sample-entry": "000000000000000000000019"
-        },
-        {
-          "name": "parampatternId",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string",
-            "pattern": "^prefix-gibberish-.+"
-          },
-          "examples": {
-            "example #0": {
-              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
-            },
-            "example #1": {
-              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
-            },
-            "example #2": {
-              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
-            },
-            "example #3": {
-              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
-            },
-            "example #4": {
-              "value": "prefix-gibberish-afterwards"
-            }
-          },
-          "x-sample-entry": "000000000000000000000019"
-        }
-      ]
-    },
-    "/{Id}": {
-      "get": {
-        "summary": "/{Id}",
-        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds",
-        "operationId": "<UUID4>",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": null,
-                "x-sample-entry": "000000000000000000000003"
-              }
-            },
-            "x-sample-entry": "000000000000000000000003"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750579.74,
-            "lastSeen": 1567750579.74,
-            "sumRT": 0.63,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750579.74,
-          "lastSeen": 1567750579.74,
-          "sumRT": 0.63,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750579.74,
-        "x-sample-entry": "000000000000000000000003"
-      },
-      "parameters": [
-        {
-          "name": "Id",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "examples": {
-            "example #0": {
-              "value": "<UUID4>"
-            },
-            "example #1": {
-              "value": "<UUID4>"
-            }
-          },
-          "x-sample-entry": "000000000000000000000003"
-        }
-      ]
-    },
-    "/{Id}/sub1": {
-      "get": {
-        "summary": "/{Id}/sub1",
-        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.11 seconds",
-        "operationId": "<UUID4>",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "text/html": {
-                "x-sample-entry": "000000000000000000000001"
-              }
-            },
-            "x-sample-entry": "000000000000000000000001"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750483.86,
-            "lastSeen": 1567750483.86,
-            "sumRT": 0.11,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750483.86,
-          "lastSeen": 1567750483.86,
-          "sumRT": 0.11,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750483.86,
-        "x-sample-entry": "000000000000000000000001"
-      },
-      "parameters": [
-        {
-          "name": "Id",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "examples": {
-            "example #0": {
-              "value": "<UUID4>"
-            },
-            "example #1": {
-              "value": "<UUID4>"
-            }
-          },
-          "x-sample-entry": "000000000000000000000003"
-        }
-      ]
-    },
-    "/{Id}/sub2": {
-      "get": {
-        "summary": "/{Id}/sub2",
-        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds",
-        "operationId": "<UUID4>",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": null,
-                "x-sample-entry": "000000000000000000000002"
-              }
-            },
-            "x-sample-entry": "000000000000000000000002"
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750578.74,
-            "lastSeen": 1567750578.74,
-            "sumRT": 0.63,
-            "sumDuration": 0
-          }
-        },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750578.74,
-          "lastSeen": 1567750578.74,
-          "sumRT": 0.63,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750578.74,
-        "x-sample-entry": "000000000000000000000002"
-      },
-      "parameters": [
-        {
-          "name": "Id",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "examples": {
-            "example #0": {
-              "value": "<UUID4>"
-            },
-            "example #1": {
-              "value": "<UUID4>"
-            }
-          },
-          "x-sample-entry": "000000000000000000000003"
-        }
-      ]
-    }
-  },
-  "x-counters-per-source": {
-    "": {
-      "entries": 19,
-      "failures": 0,
-      "firstSeen": 1567750483.86,
-      "lastSeen": 1567750582.74,
-      "sumRT": 3.27,
-      "sumDuration": 2.01
-    }
-  },
-  "x-counters-total": {
-    "entries": 19,
-    "failures": 0,
-    "firstSeen": 1567750483.86,
-    "lastSeen": 1567750582.74,
-    "sumRT": 3.27,
-    "sumDuration": 2.01
-  }
-}

agent/pkg/oas/test_artifacts/trcc.json

@@ -1,50 +0,0 @@
-{
-  "openapi": "3.1.0",
-  "info": {
-    "title": "Preloaded TRCC",
-    "version": "0.1",
-    "description": "Test file for loading pre-existing OAS"
-  },
-  "paths": {
-    "/models/{id}": {
-      "parameters": [
-        {
-          "name": "id",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string",
-            "pattern": ".+(_|-|\\.).+"
-          },
-          "example": "some-uuid-maybe"
-        }
-      ]
-    },
-    "/models/{id}/{id2}": {
-      "parameters": [
-        {
-          "name": "id",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string",
-            "pattern": ".+(_|-|\\.).+"
-          },
-          "example": "some-uuid-maybe"
-        },
-        {
-          "name": "id2",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string",
-            "pattern": "\\d+"
-          }
-        }
-      ]
-    }
-  }
-}

agent/pkg/oas/tree.go

@@ -1,58 +1,42 @@
 package oas
 
 import (
-	"encoding/json"
-	"net/url"
-	"regexp"
-	"strconv"
-	"strings"
-
 	"github.com/chanced/openapi"
 	"github.com/up9inc/mizu/shared/logger"
+	"strconv"
+	"strings"
 )
 
 type NodePath = []string
 
 type Node struct {
-	constant  *string
-	pathParam *openapi.ParameterObj
-	pathObj   *openapi.PathObj
-	parent    *Node
-	children  []*Node
+	constant *string
+	param    *openapi.ParameterObj
+	ops      *openapi.PathObj
+	parent   *Node
+	children []*Node
 }
 
-func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj, sampleId string) (node *Node) {
-	if existingPathObj == nil {
+func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Node) {
+	if pathObjToSet == nil {
 		panic("Invalid function call")
 	}
 
 	pathChunk := path[0]
-	potentialMatrix := strings.SplitN(pathChunk, ";", 2)
-	if len(potentialMatrix) > 1 {
-		pathChunk = potentialMatrix[0]
-		logger.Log.Warningf("URI matrix params are not supported: %s", potentialMatrix[1])
-	}
-
 	chunkIsParam := strings.HasPrefix(pathChunk, "{") && strings.HasSuffix(pathChunk, "}")
-	pathChunk, err := url.PathUnescape(pathChunk)
-	if err != nil {
-		logger.Log.Warningf("URI segment is not correctly encoded: %s", pathChunk)
-		// any side effects on continuing?
-	}
-
 	chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk)
 
 	var paramObj *openapi.ParameterObj
-	if chunkIsParam && existingPathObj != nil && existingPathObj.Parameters != nil {
-		_, paramObj = findParamByName(existingPathObj.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
+	if chunkIsParam && pathObjToSet != nil {
+		paramObj = findParamByName(pathObjToSet.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
 	}
 
 	if paramObj == nil {
 		node = n.searchInConstants(pathChunk)
 	}
 
-	if node == nil && pathChunk != "" {
-		node = n.searchInParams(paramObj, pathChunk, chunkIsGibberish)
+	if node == nil {
+		node = n.searchInParams(paramObj, chunkIsGibberish)
 	}
 
 	// still no node found, should create it
@@ -62,147 +46,77 @@ func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj, sampleI
 		n.children = append(n.children, node)
 
 		if paramObj != nil {
-			node.pathParam = paramObj
+			node.param = paramObj
 		} else if chunkIsGibberish {
+			initParams(&pathObjToSet.Parameters)
+
 			newParam := n.createParam()
-			node.pathParam = newParam
+			node.param = newParam
+
+			appended := append(*pathObjToSet.Parameters, newParam)
+			pathObjToSet.Parameters = &appended
 		} else {
 			node.constant = &pathChunk
 		}
 	}
 
-	if node.pathParam != nil {
-		setSampleID(&node.pathParam.Extensions, sampleId)
-	}
-
-	// add example if it's a gibberish chunk
-	if node.pathParam != nil && !chunkIsParam {
-		exmp := &node.pathParam.Examples
+	// add example if it's a param
+	if node.param != nil && !chunkIsParam {
+		exmp := &node.param.Examples
 		err := fillParamExample(&exmp, pathChunk)
 		if err != nil {
 			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
 		}
-
-		if len(*exmp) >= 3 && node.pathParam.Schema.Pattern == nil { // is it enough to decide on 2 samples?
-			node.pathParam.Schema.Pattern = getPatternFromExamples(exmp)
-		}
 	}
 
-	// TODO: eat up trailing slash, in a smart way: node.pathObj!=nil && path[1]==""
+	// TODO: eat up trailing slash, in a smart way: node.ops!=nil && path[1]==""
 	if len(path) > 1 {
-		return node.getOrSet(path[1:], existingPathObj, sampleId)
-	} else if node.pathObj == nil {
-		node.pathObj = existingPathObj
+		return node.getOrSet(path[1:], pathObjToSet)
+	} else if node.ops == nil {
+		node.ops = pathObjToSet
 	}
 
 	return node
 }
 
-func getPatternFromExamples(exmp *openapi.Examples) *openapi.Regexp {
-	allInts := true
-	strs := make([]string, 0)
-	for _, example := range *exmp {
-		exampleObj, err := example.ResolveExample(exampleResolver)
-		if err != nil {
-			continue
-		}
-
-		var value string
-		err = json.Unmarshal(exampleObj.Value, &value)
-		if err != nil {
-			logger.Log.Warningf("Failed decoding parameter example into string: %s", err)
-			continue
-		}
-		strs = append(strs, value)
-
-		if _, err := strconv.Atoi(value); err != nil {
-			allInts = false
-		}
-	}
-
-	if allInts {
-		re := new(openapi.Regexp)
-		re.Regexp = regexp.MustCompile(`\d+`)
-		return re
-	} else {
-		prefix := longestCommonXfixStr(strs, true)
-		suffix := longestCommonXfixStr(strs, false)
-
-		pat := ""
-		separators := "-._/:|*,+" // TODO: we could also cut prefix till the last separator
-		if len(prefix) > 0 && strings.Contains(separators, string(prefix[len(prefix)-1])) {
-			pat = "^" + regexp.QuoteMeta(prefix)
-		}
-
-		pat += ".+"
-
-		if len(suffix) > 0 && strings.Contains(separators, string(suffix[0])) {
-			pat += regexp.QuoteMeta(suffix) + "$"
-		}
-
-		if pat != ".+" {
-			re := new(openapi.Regexp)
-			re.Regexp = regexp.MustCompile(pat)
-			return re
-		}
-	}
-	return nil
-}
-
 func (n *Node) createParam() *openapi.ParameterObj {
 	name := "param"
 
-	if n.constant != nil { // the node is already a param
-		// REST assumption, not always correct
-		if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
-			name = *n.constant
-			name = name[:len(name)-2] + "Id"
-		} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
-			name = *n.constant
-			name = name[:len(name)-1] + "Id"
-		} else {
-			name = *n.constant + "Id"
-		}
-
-		name = cleanStr(name, isAlNumRune)
-		if !isAlphaRune(rune(name[0])) {
-			name = "_" + name
-		}
+	// REST assumption, not always correct
+	if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
+		name = *n.constant
+		name = name[:len(name)-2] + "Id"
+	} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
+		name = *n.constant
+		name = name[:len(name)-1] + "Id"
 	}
 
 	newParam := createSimpleParam(name, "path", "string")
 	x := n.countParentParams()
-	if x > 0 {
+	if x > 1 {
 		newParam.Name = newParam.Name + strconv.Itoa(x)
 	}
 
 	return newParam
 }
 
-func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunk string, chunkIsGibberish bool) *Node {
+func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunkIsGibberish bool) *Node {
 	// look among params
-	for _, subnode := range n.children {
-		if subnode.constant != nil {
-			continue
-		}
-
-		if paramObj != nil {
-			// TODO: mergeParam(subnode.pathParam, paramObj)
-			return subnode
-		} else if subnode.pathParam.Schema.Pattern != nil { // it has defined param pattern, have to respect it
-			// TODO: and not in exceptions
-			if subnode.pathParam.Schema.Pattern.Match([]byte(chunk)) {
-				return subnode
-			} else if chunkIsGibberish {
-				// TODO: what to do if gibberish chunk does not match the pattern and not in exceptions?
-				return nil
-			} else {
-				return nil
+	if paramObj != nil || chunkIsGibberish {
+		for _, subnode := range n.children {
+			if subnode.constant != nil {
+				continue
 			}
-		} else if chunkIsGibberish {
-			return subnode
-		}
 
+			// TODO: check the regex pattern of param? for exceptions etc
+
+			if paramObj != nil {
+				// TODO: mergeParam(subnode.param, paramObj)
+				return subnode
+			} else {
+				return subnode
+			}
+		}
 	}
 	return nil
 }
@@ -231,14 +145,15 @@ func (n *Node) listPaths() *openapi.Paths {
 	var strChunk string
 	if n.constant != nil {
 		strChunk = *n.constant
-	} else if n.pathParam != nil {
-		strChunk = "{" + n.pathParam.Name + "}"
-	} // else -> this is the root node
+	} else if n.param != nil {
+		strChunk = "{" + n.param.Name + "}"
+	} else {
+		// this is the root node
+	}
 
 	// add self
-	if n.pathObj != nil {
-		fillPathParams(n, n.pathObj)
-		paths.Items[openapi.PathValue(strChunk)] = n.pathObj
+	if n.ops != nil {
+		paths.Items[openapi.PathValue(strChunk)] = n.ops
 	}
 
 	// recurse into children
@@ -258,29 +173,6 @@ func (n *Node) listPaths() *openapi.Paths {
 	return paths
 }
 
-func fillPathParams(n *Node, pathObj *openapi.PathObj) {
-	// collect all path parameters from parent hierarchy
-	node := n
-	for {
-		if node.pathParam != nil {
-			initParams(&pathObj.Parameters)
-
-			idx, paramObj := findParamByName(pathObj.Parameters, openapi.InPath, node.pathParam.Name)
-			if paramObj == nil {
-				appended := append(*pathObj.Parameters, node.pathParam)
-				pathObj.Parameters = &appended
-			} else {
-				(*pathObj.Parameters)[idx] = paramObj
-			}
-		}
-
-		node = node.parent
-		if node == nil {
-			break
-		}
-	}
-}
-
 type PathAndOp struct {
 	path string
 	op   *openapi.Operation
@@ -300,7 +192,7 @@ func (n *Node) countParentParams() int {
 	res := 0
 	node := n
 	for {
-		if node.pathParam != nil {
+		if node.param != nil {
 			res++
 		}
 

agent/pkg/oas/tree_test.go

@@ -1,39 +0,0 @@
-package oas
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-
-	"github.com/chanced/openapi"
-)
-
-func TestTree(t *testing.T) {
-	testCases := []struct {
-		inp       string
-		numParams int
-		label     string
-	}{
-		{"/", 0, ""},
-		{"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation", 1, "vendor,init,conversation"},
-		{"/v1.0.0/config/launcher/sp_nKNHCzsN/{f34efcae-6583-11eb-908a-00b0fcb9d4f6}/vendor,init,conversation", 0, "vendor,init,conversation"},
-		{"/getSvgs/size/small/brand/SFLY/layoutId/170943/layoutVersion/1/sizeId/742/surface/0/isLandscape/true/childSkus/%7B%7D", 1, "{}"},
-	}
-
-	tree := new(Node)
-	for i, tc := range testCases {
-		split := strings.Split(tc.inp, "/")
-		pathObj := new(openapi.PathObj)
-		node := tree.getOrSet(split, pathObj, fmt.Sprintf("%024d", i))
-
-		fillPathParams(node, pathObj)
-
-		if node.constant != nil && *node.constant != tc.label {
-			t.Errorf("Constant does not match: %s != %s", *node.constant, tc.label)
-		}
-
-		if tc.numParams > 0 && (pathObj.Parameters == nil || len(*pathObj.Parameters) < tc.numParams) {
-			t.Errorf("Wrong num of params, expected: %d", tc.numParams)
-		}
-	}
-}

agent/pkg/oas/utils.go

@@ -3,13 +3,11 @@ package oas
 import (
 	"encoding/json"
 	"errors"
+	"github.com/chanced/openapi"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
 	"strconv"
 	"strings"
-
-	"github.com/up9inc/mizu/agent/pkg/har"
-
-	"github.com/chanced/openapi"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 func exampleResolver(ref string) (*openapi.ExampleObj, error) {
@@ -34,14 +32,16 @@ func headerResolver(ref string) (*openapi.HeaderObj, error) {
 
 func initParams(obj **openapi.ParameterList) {
 	if *obj == nil {
-		var params openapi.ParameterList = make([]openapi.Parameter, 0)
+		var params openapi.ParameterList
+		params = make([]openapi.Parameter, 0)
 		*obj = &params
 	}
 }
 
 func initHeaders(respObj *openapi.ResponseObj) {
 	if respObj.Headers == nil {
-		var created openapi.Headers = map[string]openapi.Header{}
+		var created openapi.Headers
+		created = map[string]openapi.Header{}
 		respObj.Headers = created
 	}
 }
@@ -52,7 +52,8 @@ func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *op
 	}
 	required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
 	schema := new(openapi.SchemaObj)
-	schema.Type = openapi.Types{ptype}
+	schema.Type = make(openapi.Types, 0)
+	schema.Type = append(schema.Type, ptype)
 
 	style := openapi.StyleSimple
 	if in == openapi.InQuery {
@@ -70,10 +71,9 @@ func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *op
 	return &newParam
 }
 
-func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (idx int, pathParam *openapi.ParameterObj) {
+func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (pathParam *openapi.ParameterObj) {
 	caseInsensitive := in == openapi.InHeader
-	for i, param := range *params {
-		idx = i
+	for _, param := range *params {
 		paramObj, err := param.ResolveParameter(paramResolver)
 		if err != nil {
 			logger.Log.Warningf("Failed to resolve reference: %s", err)
@@ -84,13 +84,12 @@ func findParamByName(params *openapi.ParameterList, in openapi.In, name string)
 			continue
 		}
 
-		if paramObj.Name == name || (caseInsensitive && strings.EqualFold(paramObj.Name, name)) {
+		if paramObj.Name == name || (caseInsensitive && strings.ToLower(paramObj.Name) == strings.ToLower(name)) {
 			pathParam = paramObj
 			break
 		}
 	}
-
-	return idx, pathParam
+	return pathParam
 }
 
 func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj {
@@ -101,31 +100,52 @@ func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj
 			continue
 		}
 
-		if strings.EqualFold(hname, name) {
+		if strings.ToLower(hname) == strings.ToLower(name) {
 			return hdrObj
 		}
 	}
 	return nil
 }
 
+type NVPair struct {
+	Name  string
+	Value string
+}
+
 type nvParams struct {
 	In             openapi.In
-	Pairs          []har.NVP
+	Pairs          func() []NVPair
 	IsIgnored      func(name string) bool
 	GeneralizeName func(name string) string
 }
 
-func handleNameVals(gw nvParams, params **openapi.ParameterList, checkIgnore bool, sampleId string) {
+func qstrToNVP(list []har.QueryString) []NVPair {
+	res := make([]NVPair, len(list))
+	for idx, val := range list {
+		res[idx] = NVPair{Name: val.Name, Value: val.Value}
+	}
+	return res
+}
+
+func hdrToNVP(list []har.Header) []NVPair {
+	res := make([]NVPair, len(list))
+	for idx, val := range list {
+		res[idx] = NVPair{Name: val.Name, Value: val.Value}
+	}
+	return res
+}
+
+func handleNameVals(gw nvParams, params **openapi.ParameterList) {
 	visited := map[string]*openapi.ParameterObj{}
-	for _, pair := range gw.Pairs {
-		if (checkIgnore && gw.IsIgnored(pair.Name)) || pair.Name == "" {
+	for _, pair := range gw.Pairs() {
+		if gw.IsIgnored(pair.Name) {
 			continue
 		}
 
 		nameGeneral := gw.GeneralizeName(pair.Name)
 
 		initParams(params)
-		_, param := findParamByName(*params, gw.In, pair.Name)
+		param := findParamByName(*params, gw.In, pair.Name)
 		if param == nil {
 			param = createSimpleParam(nameGeneral, gw.In, openapi.TypeString)
 			appended := append(**params, param)
@@ -137,8 +157,6 @@ func handleNameVals(gw nvParams, params **openapi.ParameterList, checkIgnore boo
 			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
 		}
 		visited[nameGeneral] = param
-
-		setSampleID(&param.Extensions, sampleId)
 	}
 
 	// maintain "required" flag
@@ -198,7 +216,7 @@ func fillParamExample(param **openapi.Examples, exampleValue string) error {
 			continue
 		}
 
-		if value == exampleValue || cnt >= 5 { // 5 examples is enough
+		if value == exampleValue || cnt > 5 { // 5 examples is enough
 			return nil
 		}
 	}
@@ -214,36 +232,6 @@ func fillParamExample(param **openapi.Examples, exampleValue string) error {
 	return nil
 }
 
-// TODO: somehow generalize the two example setting functions, plus add body example handling
-
-func addSchemaExample(existing *openapi.SchemaObj, bodyStr string) {
-	if len(existing.Examples) < 5 {
-		found := false
-		for _, eVal := range existing.Examples {
-			existingExample := ""
-			err := json.Unmarshal(eVal, &existingExample)
-			if err != nil {
-				logger.Log.Debugf("Failed to unmarshal example: %v", eVal)
-				continue
-			}
-
-			if existingExample == bodyStr {
-				found = true
-				break
-			}
-		}
-
-		if !found {
-			example, err := json.Marshal(bodyStr)
-			if err != nil {
-				logger.Log.Debugf("Failed to marshal example: %v", bodyStr)
-				return
-			}
-			existing.Examples = append(existing.Examples, example)
-		}
-	}
-}
-
 func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.com/jpillora/longestcommon
 	empty := make([]string, 0)
 	//short-circuit empty list
@@ -292,69 +280,6 @@ func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.c
 	return xfix
 }
 
-func longestCommonXfixStr(strs []string, pre bool) string { // https://github.com/jpillora/longestcommon
-	//short-circuit empty list
-	if len(strs) == 0 {
-		return ""
-	}
-	xfix := strs[0]
-	//short-circuit single-element list
-	if len(strs) == 1 {
-		return xfix
-	}
-	//compare first to rest
-	for _, str := range strs[1:] {
-		xfixl := len(xfix)
-		strl := len(str)
-		//short-circuit empty strings
-		if xfixl == 0 || strl == 0 {
-			return ""
-		}
-		//maximum possible length
-		maxl := xfixl
-		if strl < maxl {
-			maxl = strl
-		}
-		//compare letters
-		if pre {
-			//prefix, iterate left to right
-			for i := 0; i < maxl; i++ {
-				if xfix[i] != str[i] {
-					xfix = xfix[:i]
-					break
-				}
-			}
-		} else {
-			//suffix, iternate right to left
-			for i := 0; i < maxl; i++ {
-				xi := xfixl - i - 1
-				si := strl - i - 1
-				if xfix[xi] != str[si] {
-					xfix = xfix[xi+1:]
-					break
-				}
-			}
-		}
-	}
-	return xfix
-}
-
-func getSimilarPrefix(strs []string) string {
-	chunked := make([][]string, 0)
-	for _, item := range strs {
-		chunked = append(chunked, strings.Split(item, "/"))
-	}
-
-	cmn := longestCommonXfix(chunked, true)
-	res := make([]string, 0)
-	for _, chunk := range cmn {
-		if chunk != "api" && !IsVersionString(chunk) && !strings.HasPrefix(chunk, "{") {
-			res = append(res, chunk)
-		}
-	}
-	return strings.Join(res[1:], ".")
-}
-
 // returns all non-nil ops in PathObj
 func getOps(pathObj *openapi.PathObj) []*openapi.Operation {
 	ops := []**openapi.Operation{&pathObj.Get, &pathObj.Patch, &pathObj.Put, &pathObj.Options, &pathObj.Post, &pathObj.Trace, &pathObj.Head, &pathObj.Delete}
@@ -417,74 +342,3 @@ func anyJSON(text string) (anyVal interface{}, isJSON bool) {
 
 	return nil, false
 }
-
-func cleanStr(str string, criterion func(r rune) bool) string {
-	s := []byte(str)
-	j := 0
-	for _, b := range s {
-		if criterion(rune(b)) {
-			s[j] = b
-			j++
-		}
-	}
-	return string(s[:j])
-}
-
-/*
-func isAlpha(s string) bool {
-	for _, r := range s {
-		if isAlphaRune(r) {
-			return false
-		}
-	}
-	return true
-}
-*/
-
-func isAlphaRune(r rune) bool {
-	return !((r < 'a' || r > 'z') && (r < 'A' || r > 'Z'))
-}
-
-func isAlNumRune(b rune) bool {
-	return isAlphaRune(b) || ('0' <= b && b <= '9')
-}
-
-func deleteFromSlice(s []string, val string) []string {
-	temp := s[:0]
-	for _, x := range s {
-		if x != val {
-			temp = append(temp, x)
-		}
-	}
-	return temp
-}
-
-func sliceContains(s []string, e string) bool {
-	for _, a := range s {
-		if a == e {
-			return true
-		}
-	}
-	return false
-}
-
-func intersectSliceWithMap(required []string, names map[string]struct{}) []string {
-	for name := range names {
-		if !sliceContains(required, name) {
-			required = deleteFromSlice(required, name)
-		}
-	}
-	return required
-}
-
-func setSampleID(extensions *openapi.Extensions, id string) {
-	if id != "" {
-		if *extensions == nil {
-			*extensions = openapi.Extensions{}
-		}
-		err := (extensions).SetExtension(SampleId, id)
-		if err != nil {
-			logger.Log.Warningf("Failed to set sample ID: %s", err)
-		}
-	}
-}

agent/pkg/oas/utils_test.go

@@ -29,31 +29,7 @@ func TestAnyJSON(t *testing.T) {
 		} else if tc.inp == "null" && any != nil {
 			t.Errorf("null has to parse as nil (but got %s)", any)
 		} else {
-			t.Logf("%s => %v", tc.inp, any)
+			t.Logf("%s => %s", tc.inp, any)
 		}
 	}
 }
-
-func TestStrRunes(t *testing.T) {
-	if isAlphaRune('5') {
-		t.Logf("Failed")
-	}
-	if !isAlphaRune('a') {
-		t.Logf("Failed")
-	}
-
-	if !isAlNumRune('5') {
-		t.Logf("Failed")
-	}
-	if isAlNumRune(' ') {
-		t.Logf("Failed")
-	}
-
-	if cleanStr("-abc_567", isAlphaRune) != "abc" {
-		t.Logf("Failed")
-	}
-
-	if cleanStr("-abc_567", isAlNumRune) != "abc567" {
-		t.Logf("Failed")
-	}
-}

agent/pkg/providers/install_provider.go

@@ -0,0 +1,18 @@
+package providers
+
+import (
+	"context"
+	"mizuserver/pkg/config"
+)
+
+func IsInstallNeeded() (bool, error) {
+	if !config.Config.StandaloneMode { // install not needed in ephermeral mizu
+		return false, nil
+	}
+
+	if anyUserExists, err := AnyUserExists(context.Background()); err != nil {
+		return false, err
+	} else {
+		return !anyUserExists, nil
+	}
+}

agent/pkg/providers/kubernetes_provider.go

@@ -0,0 +1,27 @@
+package providers
+
+import (
+	"github.com/up9inc/mizu/shared/kubernetes"
+	"sync"
+)
+
+var lock = &sync.Mutex{}
+
+var kubernetesProvider *kubernetes.Provider
+
+func GetKubernetesProvider() (*kubernetes.Provider, error) {
+	if kubernetesProvider == nil {
+		lock.Lock()
+		defer lock.Unlock()
+
+		if kubernetesProvider == nil {
+			var err error
+			kubernetesProvider, err = kubernetes.NewProviderInCluster()
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return kubernetesProvider, nil
+}

agent/pkg/providers/stats_provider.go

@@ -7,7 +7,6 @@ import (
 
 type GeneralStats struct {
 	EntriesCount        int
-	EntriesVolumeInGB   float64
 	FirstEntryTimestamp int
 	LastEntryTimestamp  int
 }
@@ -22,9 +21,8 @@ func GetGeneralStats() GeneralStats {
 	return generalStats
 }
 
-func EntryAdded(size int) {
+func EntryAdded() {
 	generalStats.EntriesCount++
-	generalStats.EntriesVolumeInGB += float64(size) / (1 << 30)
 
 	currentTimestamp := int(time.Now().Unix())
 
@@ -34,3 +32,5 @@ func EntryAdded(size int) {
 
 	generalStats.LastEntryTimestamp = currentTimestamp
 }
+
+

agent/pkg/providers/stats_provider_test.go

@@ -2,9 +2,8 @@ package providers_test
 
 import (
 	"fmt"
+	"mizuserver/pkg/providers"
 	"testing"
-
-	"github.com/up9inc/mizu/agent/pkg/providers"
 )
 
 func TestNoEntryAddedCount(t *testing.T) {
@@ -13,10 +12,6 @@ func TestNoEntryAddedCount(t *testing.T) {
 	if entriesStats.EntriesCount != 0 {
 		t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesCount)
 	}
-
-	if entriesStats.EntriesVolumeInGB != 0 {
-		t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesVolumeInGB)
-	}
 }
 
 func TestEntryAddedCount(t *testing.T) {
@@ -25,7 +20,7 @@ func TestEntryAddedCount(t *testing.T) {
 	for _, entriesCount := range tests {
 		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
 			for i := 0; i < entriesCount; i++ {
-				providers.EntryAdded(0)
+				providers.EntryAdded()
 			}
 
 			entriesStats := providers.GetGeneralStats()
@@ -34,38 +29,7 @@ func TestEntryAddedCount(t *testing.T) {
 				t.Errorf("unexpected result - expected: %v, actual: %v", entriesCount, entriesStats.EntriesCount)
 			}
 
-			if entriesStats.EntriesVolumeInGB != 0 {
-				t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesVolumeInGB)
-			}
-
 			t.Cleanup(providers.ResetGeneralStats)
 		})
 	}
 }
-
-func TestEntryAddedVolume(t *testing.T) {
-	// 6 bytes + 4 bytes
-	tests := [][]byte{[]byte("volume"), []byte("test")}
-	var expectedEntriesCount int
-	var expectedVolumeInGB float64
-
-	for _, data := range tests {
-		t.Run(fmt.Sprintf("%d", len(data)), func(t *testing.T) {
-			expectedEntriesCount++
-			expectedVolumeInGB += float64(len(data)) / (1 << 30)
-
-			providers.EntryAdded(len(data))
-
-			entriesStats := providers.GetGeneralStats()
-
-			if entriesStats.EntriesCount != expectedEntriesCount {
-				t.Errorf("unexpected result - expected: %v, actual: %v", expectedEntriesCount, entriesStats.EntriesCount)
-			}
-
-			if entriesStats.EntriesVolumeInGB != expectedVolumeInGB {
-				t.Errorf("unexpected result - expected: %v, actual: %v", expectedVolumeInGB, entriesStats.EntriesVolumeInGB)
-			}
-		})
-	}
-
-}