Merge branch 'develop' into feat/bfl-syntax-highlighting

This commit is contained in:
M. Mert Yildiran 2022-07-28 02:16:41 +03:00
commit f8b0ffd72c
No known key found for this signature in database
GPG Key ID: D42ADB236521BF7A
617 changed files with 493562 additions and 39523 deletions

View File

@ -2,7 +2,6 @@
.dockerignore
.editorconfig
.gitignore
**/.env*
Dockerfile
Makefile
LICENSE

View File

@ -8,37 +8,47 @@ on:
branches:
- 'develop'
concurrency:
group: mizu-acceptance-tests-${{ github.ref }}
cancel-in-progress: true
env:
MIZU_CI_IMAGE: mizu/ci:0.0
jobs:
run-acceptance-tests:
name: Run acceptance tests
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.16
- name: Set up Go 1.17
uses: actions/setup-go@v2
with:
go-version: '^1.16'
go-version: '^1.17'
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Build
uses: docker/build-push-action@v2
with:
context: .
push: false
load: true
tags: ${{ env.MIZU_CI_IMAGE }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Setup acceptance test
run: source ./acceptanceTests/setup.sh
run: ./acceptanceTests/setup.sh
- name: Create k8s users and change context
env:
USERNAME_UNRESTRICTED: user-with-clusterwide-access
USERNAME_RESTRICTED: user-with-restricted-access
run: |
./acceptanceTests/create_user.sh "${USERNAME_UNRESTRICTED}"
./acceptanceTests/create_user.sh "${USERNAME_RESTRICTED}"
kubectl apply -f cli/cmd/permissionFiles/permissions-all-namespaces-tap.yaml
kubectl config use-context ${USERNAME_UNRESTRICTED}
- name: Test
run: make acceptance-test
- name: Slack notification on failure
uses: ravsamhq/notify-slack-action@v1
if: always()
with:
status: ${{ job.status }}
notification_title: 'Mizu {workflow} has {status_message}'
message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit: <{commit_url}|{commit_sha}>'
footer: 'Linked Repo <{repo_url}|{repo}>'
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -0,0 +1,54 @@
name: Acceptance tests on PR
on: push
env:
MIZU_CI_IMAGE: mizu/ci:0.0
concurrency:
group: acceptance-tests-on-pr-${{ github.ref }}
cancel-in-progress: true
jobs:
run-tests:
name: Run tests
runs-on: ubuntu-latest
if: ${{ contains(github.event.head_commit.message, '#run_acceptance_tests') }}
steps:
- name: Set up Go 1.17
uses: actions/setup-go@v2
with:
go-version: '^1.17'
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Build
uses: docker/build-push-action@v2
with:
context: .
push: false
load: true
tags: ${{ env.MIZU_CI_IMAGE }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Setup acceptance test
run: ./acceptanceTests/setup.sh
- name: Create k8s users and change context
env:
USERNAME_UNRESTRICTED: user-with-clusterwide-access
USERNAME_RESTRICTED: user-with-restricted-access
run: |
./acceptanceTests/create_user.sh "${USERNAME_UNRESTRICTED}"
./acceptanceTests/create_user.sh "${USERNAME_RESTRICTED}"
kubectl apply -f cli/cmd/permissionFiles/permissions-all-namespaces-tap.yaml
kubectl config use-context ${USERNAME_UNRESTRICTED}
- name: Test
run: make acceptance-test

View File

@ -0,0 +1,44 @@
name: Build Custom Branch
on: push
concurrency:
group: custom-branch-build-${{ github.ref }}
cancel-in-progress: true
jobs:
build:
name: Push custom branch image to GCR
runs-on: ubuntu-latest
if: ${{ contains(github.event.head_commit.message, '#build_and_publish_custom_image') }}
steps:
- name: Check out the repo
uses: actions/checkout@v2
- id: 'auth'
uses: 'google-github-actions/auth@v0'
with:
credentials_json: '${{ secrets.GCR_JSON_KEY }}'
- name: 'Set up Cloud SDK'
uses: 'google-github-actions/setup-gcloud@v0'
- name: Get base image name
shell: bash
run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
id: base_image_step
- name: Login to GCR
uses: docker/login-action@v1
with:
registry: gcr.io
username: _json_key
password: ${{ secrets.GCR_JSON_KEY }}
- name: Build and push
uses: docker/build-push-action@v2
with:
context: .
push: true
tags: ${{ steps.base_image_step.outputs.image }}:latest

62
.github/workflows/build.yml vendored Normal file
View File

@ -0,0 +1,62 @@
name: Build
on:
pull_request:
branches:
- 'develop'
- 'main'
concurrency:
group: mizu-pr-validation-${{ github.ref }}
cancel-in-progress: true
jobs:
build-cli:
name: CLI executable build
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
fetch-depth: 2
- name: Check modified files
id: modified_files
run: devops/check_modified_files.sh cli/
- name: Set up Go 1.17
if: steps.modified_files.outputs.matched == 'true'
uses: actions/setup-go@v2
with:
go-version: '1.17'
- name: Build CLI
if: steps.modified_files.outputs.matched == 'true'
run: make cli
build-agent:
name: Agent docker image build
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
fetch-depth: 2
- name: Check modified files
id: modified_files
run: devops/check_modified_files.sh agent/ shared/ tap/ ui/ ui-common/ Dockerfile
- name: Set up Docker Buildx
if: steps.modified_files.outputs.matched == 'true'
uses: docker/setup-buildx-action@v1
- name: Build
uses: docker/build-push-action@v2
if: steps.modified_files.outputs.matched == 'true'
with:
context: .
push: false
tags: up9inc/mizu:devlatest
cache-from: type=gha
cache-to: type=gha,mode=max

View File

@ -13,10 +13,11 @@ jobs:
- uses: actions/stale@v3
with:
days-before-issue-stale: 30
days-before-issue-close: 14
days-before-issue-close: -1
exempt-issue-labels: "enhancement"
stale-issue-label: "stale"
stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
close-issue-message: ""
days-before-pr-stale: -1
days-before-pr-close: -1
repo-token: ${{ secrets.GITHUB_TOKEN }}

View File

@ -1,46 +0,0 @@
name: PR validation
on:
pull_request:
branches:
- 'develop'
- 'main'
concurrency:
group: mizu-pr-validation-${{ github.ref }}
cancel-in-progress: true
jobs:
build-cli:
name: Build CLI
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.16
uses: actions/setup-go@v2
with:
go-version: '1.16'
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- name: Build CLI
run: make cli
build-agent:
name: Build Agent
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.16
uses: actions/setup-go@v2
with:
go-version: '1.16'
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- shell: bash
run: |
sudo apt-get install libpcap-dev
- name: Build Agent
run: make agent

View File

@ -1,90 +0,0 @@
name: publish
on:
push:
branches:
- 'develop'
- 'main'
concurrency:
group: mizu-publish-${{ github.ref }}
cancel-in-progress: true
jobs:
docker:
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.16
uses: actions/setup-go@v2
with:
go-version: '1.16'
- name: Checkout
uses: actions/checkout@v2
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@master
with:
service_account_key: ${{ secrets.GCR_JSON_KEY }}
export_default_credentials: true
- uses: haya14busa/action-cond@v1
id: condval
with:
cond: ${{ github.ref == 'refs/heads/main' }}
if_true: "minor"
if_false: "patch"
- name: Auto Increment Semver Action
uses: MCKanpolat/auto-semver-action@1.0.5
id: versioning
with:
releaseType: ${{ steps.condval.outputs.value }}
github_token: ${{ secrets.GITHUB_TOKEN }}
- name: Get version parameters
shell: bash
run: |
echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
id: version_parameters
- name: Get base image name
shell: bash
run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
id: base_image_step
- name: Docker meta
id: meta
uses: crazy-max/ghaction-docker-meta@v2
with:
images: ${{ steps.base_image_step.outputs.image }}
tags: |
type=sha
type=raw,${{ github.sha }}
type=raw,${{ steps.versioning.outputs.version }}
- name: Login to DockerHub
uses: docker/login-action@v1
with:
registry: gcr.io
username: _json_key
password: ${{ secrets.GCR_JSON_KEY }}
- name: Build and push
uses: docker/build-push-action@v2
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
SEM_VER=${{ steps.versioning.outputs.version }}
BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
COMMIT_HASH=${{ github.sha }}
- name: Build and Push CLI
run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
- shell: bash
run: |
echo '${{ steps.versioning.outputs.version }}' >> cli/bin/version.txt
- name: publish
uses: ncipollo/release-action@v1
with:
token: ${{ secrets.GITHUB_TOKEN }}
artifacts: "cli/bin/*"
commit: ${{ steps.version_parameters.outputs.branch }}
tag: ${{ steps.versioning.outputs.version }}
prerelease: ${{ github.ref != 'refs/heads/main' }}
bodyFile: 'cli/bin/README.md'

204
.github/workflows/release.yml vendored Normal file
View File

@ -0,0 +1,204 @@
name: Release
on:
push:
branches:
- 'develop'
- 'main'
concurrency:
group: mizu-publish-${{ github.ref }}
cancel-in-progress: true
jobs:
docker-registry:
name: Push Docker image to Docker Hub
runs-on: ubuntu-latest
strategy:
max-parallel: 2
fail-fast: false
matrix:
target:
- amd64
- arm64v8
steps:
- name: Check out the repo
uses: actions/checkout@v2
- name: Determine versioning strategy
uses: haya14busa/action-cond@v1
id: condval
with:
cond: ${{ github.ref == 'refs/heads/main' }}
if_true: "stable"
if_false: "dev"
- name: Auto Increment Ver Action
uses: docker://igorgov/auto-inc-ver:v2.0.0
id: versioning
with:
mode: ${{ steps.condval.outputs.value }}
suffix: 'dev'
github_token: ${{ secrets.GITHUB_TOKEN }}
- name: Get version parameters
shell: bash
run: |
echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
id: version_parameters
- name: Docker meta
id: meta
uses: docker/metadata-action@v3
with:
images: |
up9inc/mizu
tags: |
type=raw,${{ steps.versioning.outputs.version }}
type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
type=raw,value=dev-latest,enable=${{ steps.condval.outputs.value == 'dev' }}
flavor: |
latest=auto
prefix=
suffix=-${{ matrix.target }},onlatest=true
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_PASS }}
- name: Build and push
uses: docker/build-push-action@v2
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
TARGETARCH=${{ matrix.target }}
VER=${{ steps.versioning.outputs.version }}
BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
COMMIT_HASH=${{ github.sha }}
docker-manifest:
name: Create and Push a Docker Manifest
runs-on: ubuntu-latest
needs: [docker-registry]
steps:
- name: Determine versioning strategy
uses: haya14busa/action-cond@v1
id: condval
with:
cond: ${{ github.ref == 'refs/heads/main' }}
if_true: "stable"
if_false: "dev"
- name: Auto Increment Ver Action
uses: docker://igorgov/auto-inc-ver:v2.0.0
id: versioning
with:
mode: ${{ steps.condval.outputs.value }}
suffix: 'dev'
github_token: ${{ secrets.GITHUB_TOKEN }}
- name: Get version parameters
shell: bash
run: |
echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
id: version_parameters
- name: Docker meta
id: meta
uses: docker/metadata-action@v3
with:
images: |
up9inc/mizu
tags: |
type=raw,${{ steps.versioning.outputs.version }}
type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
type=raw,value=dev-latest,enable=${{ steps.condval.outputs.value == 'dev' }}
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_PASS }}
- name: Create manifest
run: |
while IFS= read -r line; do
docker manifest create $line --amend $line-amd64 --amend $line-arm64v8
done <<< "${{ steps.meta.outputs.tags }}"
- name: Push manifest
run: |
while IFS= read -r line; do
docker manifest push $line
done <<< "${{ steps.meta.outputs.tags }}"
cli:
name: Build the CLI and publish
runs-on: ubuntu-latest
needs: [docker-manifest]
steps:
- name: Set up Go 1.17
uses: actions/setup-go@v2
with:
go-version: '1.17'
- name: Check out the repo
uses: actions/checkout@v2
- id: 'auth'
uses: 'google-github-actions/auth@v0'
with:
credentials_json: '${{ secrets.GCR_JSON_KEY }}'
- name: 'Set up Cloud SDK'
uses: 'google-github-actions/setup-gcloud@v0'
- name: Determine versioning strategy
uses: haya14busa/action-cond@v1
id: condval
with:
cond: ${{ github.ref == 'refs/heads/main' }}
if_true: "stable"
if_false: "dev"
- name: Auto Increment Ver Action
uses: docker://igorgov/auto-inc-ver:v2.0.0
id: versioning
with:
mode: ${{ steps.condval.outputs.value }}
suffix: 'dev'
github_token: ${{ secrets.GITHUB_TOKEN }}
- name: Get version parameters
shell: bash
run: |
echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
id: version_parameters
- name: Build and Push CLI
run: make push-cli VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
- name: Log the version into a .txt file
shell: bash
run: |
echo '${{ steps.versioning.outputs.version }}' >> cli/bin/version.txt
- name: Release
uses: ncipollo/release-action@v1
with:
token: ${{ secrets.GITHUB_TOKEN }}
artifacts: "cli/bin/*"
commit: ${{ steps.version_parameters.outputs.branch }}
tag: ${{ steps.versioning.outputs.version }}
prerelease: ${{ github.ref != 'refs/heads/main' }}
bodyFile: 'cli/bin/README.md'

View File

@ -0,0 +1,206 @@
name: Static code analysis
on:
pull_request:
branches:
- 'develop'
- 'main'
permissions:
contents: read
jobs:
go-lint:
name: Go lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 2
- uses: actions/setup-go@v2
with:
go-version: '^1.17'
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y libpcap-dev
./devops/install-capstone.sh
- name: Check Agent modified files
id: agent_modified_files
run: devops/check_modified_files.sh agent/
- name: Generate eBPF object files and go bindings
id: generate_ebpf
run: make bpf
- name: Go lint - agent
uses: golangci/golangci-lint-action@v2
if: steps.agent_modified_files.outputs.matched == 'true'
with:
version: latest
working-directory: agent
args: --timeout=10m
- name: Check shared modified files
id: shared_modified_files
run: devops/check_modified_files.sh shared/
- name: Go lint - shared
uses: golangci/golangci-lint-action@v2
if: steps.shared_modified_files.outputs.matched == 'true'
with:
version: latest
working-directory: shared
args: --timeout=10m
- name: Check tap modified files
id: tap_modified_files
run: devops/check_modified_files.sh tap/
- name: Go lint - tap
uses: golangci/golangci-lint-action@v2
if: steps.tap_modified_files.outputs.matched == 'true'
with:
version: latest
working-directory: tap
args: --timeout=10m
- name: Check cli modified files
id: cli_modified_files
run: devops/check_modified_files.sh cli/
- name: Go lint - CLI
uses: golangci/golangci-lint-action@v2
if: steps.cli_modified_files.outputs.matched == 'true'
with:
version: latest
working-directory: cli
args: --timeout=10m
- name: Check acceptanceTests modified files
id: acceptanceTests_modified_files
run: devops/check_modified_files.sh acceptanceTests/
- name: Go lint - acceptanceTests
uses: golangci/golangci-lint-action@v2
if: steps.acceptanceTests_modified_files.outputs.matched == 'true'
with:
version: latest
working-directory: acceptanceTests
args: --timeout=10m
- name: Check tap/api modified files
id: tap_api_modified_files
run: devops/check_modified_files.sh tap/api/
- name: Go lint - tap/api
uses: golangci/golangci-lint-action@v2
if: steps.tap_api_modified_files.outputs.matched == 'true'
with:
version: latest
working-directory: tap/api
- name: Check tap/extensions/amqp modified files
id: tap_amqp_modified_files
run: devops/check_modified_files.sh tap/extensions/amqp/
- name: Go lint - tap/extensions/amqp
uses: golangci/golangci-lint-action@v2
if: steps.tap_amqp_modified_files.outputs.matched == 'true'
with:
version: latest
working-directory: tap/extensions/amqp
- name: Check tap/extensions/http modified files
id: tap_http_modified_files
run: devops/check_modified_files.sh tap/extensions/http/
- name: Go lint - tap/extensions/http
uses: golangci/golangci-lint-action@v2
if: steps.tap_http_modified_files.outputs.matched == 'true'
with:
version: latest
working-directory: tap/extensions/http
- name: Check tap/extensions/kafka modified files
id: tap_kafka_modified_files
run: devops/check_modified_files.sh tap/extensions/kafka/
- name: Go lint - tap/extensions/kafka
uses: golangci/golangci-lint-action@v2
if: steps.tap_kafka_modified_files.outputs.matched == 'true'
with:
version: latest
working-directory: tap/extensions/kafka
- name: Check tap/extensions/redis modified files
id: tap_redis_modified_files
run: devops/check_modified_files.sh tap/extensions/redis/
- name: Go lint - tap/extensions/redis
uses: golangci/golangci-lint-action@v2
if: steps.tap_redis_modified_files.outputs.matched == 'true'
with:
version: latest
working-directory: tap/extensions/redis
- name: Check logger modified files
id: logger_modified_files
run: devops/check_modified_files.sh logger/
- name: Go lint - logger
uses: golangci/golangci-lint-action@v2
if: steps.logger_modified_files.outputs.matched == 'true'
with:
version: latest
working-directory: logger
es-lint:
name: ES lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 2
- uses: actions/setup-node@v2
with:
node-version: 16
- name: Check modified UI files
id: ui_modified_files
run: devops/check_modified_files.sh ui/
- name: ESLint prerequisites ui
if: steps.ui_modified_files.outputs.matched == 'true'
run: |
sudo npm install -g eslint
cd ui
npm run prestart
npm i
- name: ESLint ui
if: steps.ui_modified_files.outputs.matched == 'true'
run: |
cd ui
npm run eslint
- name: Check modified ui-common files
id: ui_common_modified_files
run: devops/check_modified_files.sh ui-common/
- name: ESLint prerequisites ui-common
if: steps.ui_common_modified_files.outputs.matched == 'true'
run: |
sudo npm install -g eslint
cd ui-common
npm i
- name: ESLint ui-common
if: steps.ui_common_modified_files.outputs.matched == 'true'
run: |
cd ui-common
npm run eslint

75
.github/workflows/test.yml vendored Normal file
View File

@ -0,0 +1,75 @@
name: Test
on:
pull_request:
branches:
- 'develop'
- 'main'
push: # needed to upload test coverage report to codecov
branches:
- 'develop'
- 'main'
concurrency:
group: mizu-tests-validation-${{ github.ref }}
cancel-in-progress: true
jobs:
run-unit-tests:
name: Unit Tests
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
fetch-depth: 2
- name: Set up Go 1.17
uses: actions/setup-go@v2
with:
go-version: '^1.17'
- name: Install libpcap
shell: bash
run: |
sudo apt-get install libpcap-dev
- name: Install Capstone
shell: bash
run: |
./devops/install-capstone.sh
- name: Generate eBPF object files and go bindings
id: generate_ebpf
run: make bpf
- name: Check CLI modified files
id: cli_modified_files
run: devops/check_modified_files.sh cli/
- name: CLI Test
if: github.event_name == 'push' || steps.cli_modified_files.outputs.matched == 'true'
run: make test-cli
- name: Check Agent modified files
id: agent_modified_files
run: devops/check_modified_files.sh agent/
- name: Agent Test
if: github.event_name == 'push' || steps.agent_modified_files.outputs.matched == 'true'
run: make test-agent
- name: Shared Test
run: make test-shared
- name: Check extensions modified files
id: ext_modified_files
run: devops/check_modified_files.sh tap/extensions/ tap/api/
- name: Extensions Test
if: github.event_name == 'push' || steps.ext_modified_files.outputs.matched == 'true'
run: make test-extensions
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v2

View File

@ -1,56 +0,0 @@
name: tests validation
on:
pull_request:
branches:
- 'develop'
- 'main'
push:
branches:
- 'develop'
- 'main'
concurrency:
group: mizu-tests-validation-${{ github.ref }}
cancel-in-progress: true
jobs:
run-tests-cli:
name: Run CLI tests
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.16
uses: actions/setup-go@v2
with:
go-version: '^1.16'
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- name: Test
run: make test-cli
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v2
run-tests-agent:
name: Run Agent tests
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.16
uses: actions/setup-go@v2
with:
go-version: '^1.16'
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- shell: bash
run: |
sudo apt-get install libpcap-dev
- name: Test
run: make test-agent
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v2

17
.gitignore vendored
View File

@ -30,7 +30,7 @@ build
pprof/*
# Database Files
*.bin
*.db
*.gob
# Nohup Files - https://man7.org/linux/man-pages/man1/nohup.1p.html
@ -44,3 +44,18 @@ cypress.env.json
*/cypress/screenshots
*/cypress/videos
*/cypress/support
# Ignore test data in extensions
tap/extensions/*/bin
tap/extensions/*/expect
# UI folders to ignore
**/node_modules/**
**/dist/**
*.editorconfig
# Ignore *.log files
*.log
# Object files
*.o

View File

@ -1,63 +1,142 @@
FROM node:14-slim AS site-build
ARG BUILDARCH=amd64
ARG TARGETARCH=amd64
### Front-end common
FROM node:16 AS front-end-common
WORKDIR /app/ui-build
COPY ui-common/package.json .
COPY ui-common/package-lock.json .
RUN npm i
COPY ui-common .
RUN npm pack
### Front-end
FROM node:16 AS front-end
WORKDIR /app/ui-build
COPY ui/package.json .
COPY ui/package-lock.json .
COPY ui/package.json ui/package-lock.json ./
COPY --from=front-end-common ["/app/ui-build/up9-mizu-common-0.0.0.tgz", "."]
RUN npm i
COPY ui .
RUN npm run build
### Base builder image for native builds architecture
FROM golang:1.17-alpine AS builder-native-base
ENV CGO_ENABLED=1 GOOS=linux
RUN apk add --no-cache \
libpcap-dev \
g++ \
perl-utils \
curl \
build-base \
binutils-gold \
bash \
clang \
llvm \
libbpf-dev \
linux-headers
COPY devops/install-capstone.sh .
RUN ./install-capstone.sh
FROM golang:1.16-alpine AS builder
# Set necessary environment variables needed for our image.
ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
RUN apk add libpcap-dev gcc g++ make bash perl-utils
### Intermediate builder image for x86-64 to x86-64 native builds
FROM builder-native-base AS builder-from-amd64-to-amd64
ENV GOARCH=amd64
ENV BPF_TARGET=amd64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_x86"
# Move to agent working directory (/agent-build).
### Intermediate builder image for AArch64 to AArch64 native builds
FROM builder-native-base AS builder-from-arm64v8-to-arm64v8
ENV GOARCH=arm64
ENV BPF_TARGET=arm64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64"
### Builder image for x86-64 to AArch64 cross-compilation
FROM up9inc/linux-arm64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 AS builder-from-amd64-to-arm64v8
ENV CGO_ENABLED=1 GOOS=linux
ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap -I/work/capstone/include"
ENV BPF_TARGET=arm64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64 -I/usr/xcc/aarch64-linux-musl-cross/aarch64-linux-musl/include/"
### Builder image for AArch64 to x86-64 cross-compilation
FROM up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 AS builder-from-arm64v8-to-amd64
ENV CGO_ENABLED=1 GOOS=linux
ENV GOARCH=amd64 CGO_CFLAGS="-I/libpcap -I/capstone/include"
ENV BPF_TARGET=amd64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_x86 -I/usr/local/musl/x86_64-unknown-linux-musl/include/"
### Final builder image where the build happens
# Possible build strategies:
# BUILDARCH=amd64 TARGETARCH=amd64
# BUILDARCH=arm64v8 TARGETARCH=arm64v8
# BUILDARCH=amd64 TARGETARCH=arm64v8
# BUILDARCH=arm64v8 TARGETARCH=amd64
ARG BUILDARCH=amd64
ARG TARGETARCH=amd64
FROM builder-from-${BUILDARCH}-to-${TARGETARCH} AS builder
# Move to agent working directory (/agent-build)
WORKDIR /app/agent-build
COPY agent/go.mod agent/go.sum ./
COPY shared/go.mod shared/go.mod ../shared/
COPY logger/go.mod logger/go.mod ../logger/
COPY tap/go.mod tap/go.mod ../tap/
COPY tap/api/go.* ../tap/api/
COPY tap/api/go.mod ../tap/api/
COPY tap/dbgctl/go.mod ../tap/dbgctl/
COPY tap/extensions/amqp/go.mod ../tap/extensions/amqp/
COPY tap/extensions/http/go.mod ../tap/extensions/http/
COPY tap/extensions/kafka/go.mod ../tap/extensions/kafka/
COPY tap/extensions/redis/go.mod ../tap/extensions/redis/
RUN go mod download
# cheap trick to make the build faster (As long as go.mod wasn't changes)
RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
# Copy and build agent code
COPY shared ../shared
COPY logger ../logger
COPY tap ../tap
COPY agent .
ARG COMMIT_HASH
ARG GIT_BRANCH
ARG BUILD_TIMESTAMP
ARG SEM_VER=0.0.0
ARG VER=0.0
# Copy and build agent code
COPY shared ../shared
COPY tap ../tap
COPY agent .
RUN go build -ldflags="-s -w \
-X 'mizuserver/pkg/version.GitCommitHash=${COMMIT_HASH}' \
-X 'mizuserver/pkg/version.Branch=${GIT_BRANCH}' \
-X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
-X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
WORKDIR /app/tap/tlstapper
COPY devops/build_extensions.sh ..
RUN cd .. && /bin/bash build_extensions.sh
RUN rm tlstapper_bpf*
RUN GOARCH=${BUILDARCH} go generate tls_tapper.go
FROM alpine:3.15
WORKDIR /app/agent-build
RUN apk add bash libpcap-dev
RUN go build -ldflags="-extldflags=-static -s -w \
-X 'github.com/up9inc/mizu/agent/pkg/version.GitCommitHash=${COMMIT_HASH}' \
-X 'github.com/up9inc/mizu/agent/pkg/version.Branch=${GIT_BRANCH}' \
-X 'github.com/up9inc/mizu/agent/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
-X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
# Download Basenine executable, verify the sha1sum
ADD https://github.com/up9inc/basenine/releases/download/v0.8.3/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
ADD https://github.com/up9inc/basenine/releases/download/v0.8.3/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
RUN shasum -a 256 -c basenine_linux_"${GOARCH}".sha256 && \
chmod +x ./basenine_linux_"${GOARCH}" && \
mv ./basenine_linux_"${GOARCH}" ./basenine
### The shipped image
ARG TARGETARCH=amd64
FROM ${TARGETARCH}/busybox:latest
# gin-gonic runs in debug mode without this
ENV GIN_MODE=release
WORKDIR /app/data/
WORKDIR /app
# Copy binary and config files from /build to root folder of scratch container.
COPY --from=builder ["/app/agent-build/mizuagent", "."]
COPY --from=builder ["/app/agent/build/extensions", "extensions"]
COPY --from=site-build ["/app/ui-build/build", "site"]
RUN mkdir /app/data/
# gin-gonic runs in debug mode without this
ENV GIN_MODE=release
COPY --from=builder ["/app/agent-build/basenine", "/usr/local/bin/basenine"]
COPY --from=front-end ["/app/ui-build/build", "site"]
# this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
ENTRYPOINT "/app/mizuagent"
ENTRYPOINT ["/app/mizuagent"]

View File

@ -8,7 +8,7 @@ SHELL=/bin/bash
# HELP
# This will output the help for each task
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY: help ui extensions extensions-debug agent agent-debug cli tap docker
.PHONY: help ui agent agent-debug cli tap docker bpf clean-bpf
help: ## This help.
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@ -19,7 +19,14 @@ help: ## This help.
TS_SUFFIX="$(shell date '+%s')"
GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
export SEM_VER?=0.0.0
export VER?=0.0
ARCH=$(shell uname -m)
ifeq ($(ARCH),$(filter $(ARCH),aarch64 arm64))
BPF_O_ARCH_LABEL=arm64
else
BPF_O_ARCH_LABEL=x86
endif
BPF_O_FILES = tap/tlstapper/tlstapper46_bpfel_$(BPF_O_ARCH_LABEL).o tap/tlstapper/tlstapper_bpfel_$(BPF_O_ARCH_LABEL).o
ui: ## Build UI.
@(cd ui; npm i ; npm run build; )
@ -31,38 +38,35 @@ cli: ## Build CLI.
cli-debug: ## Build CLI.
@echo "building cli"; cd cli && $(MAKE) build-debug
build-cli-ci: ## Build CLI for CI.
@echo "building cli for ci"; cd cli && $(MAKE) build GIT_BRANCH=ci SUFFIX=ci
agent: ## Build agent.
agent: bpf ## Build agent.
@(echo "building mizu agent .." )
@(cd agent; go build -o build/mizuagent main.go)
${MAKE} extensions
@ls -l agent/build
bpf: $(BPF_O_FILES)
$(BPF_O_FILES): $(wildcard tap/tlstapper/bpf/**/*.[ch])
@(echo "building tlstapper bpf")
@(./tap/tlstapper/bpf-builder/build.sh)
agent-debug: ## Build agent for debug.
@(echo "building mizu agent for debug.." )
@(cd agent; go build -gcflags="all=-N -l" -o build/mizuagent main.go)
${MAKE} extensions-debug
@ls -l agent/build
docker: ## Build and publish agent docker image.
$(MAKE) push-docker
agent-docker: ## Build agent docker image.
@echo "Building agent docker image"
@docker build -t up9inc/mizu:devlatest .
push: push-docker push-cli ## Build and publish agent docker image & CLI.
push-docker: ## Build and publish agent docker image.
@echo "publishing Docker image .. "
devops/build-push-featurebranch.sh
push-docker-debug:
@echo "publishing debug Docker image .. "
devops/build-push-featurebranch-debug.sh
build-docker-ci: ## Build agent docker image for CI.
@echo "building docker image for ci"
devops/build-agent-ci.sh
push-cli: ## Build and publish CLI.
@echo "publishing CLI .. "
@cd cli; $(MAKE) build-all
@ -71,7 +75,7 @@ push-cli: ## Build and publish CLI.
gsutil cp -r ./cli/bin/* gs://${BUCKET_PATH}/
gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
clean: clean-ui clean-agent clean-cli clean-docker clean-extensions ## Clean all build artifacts.
clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
clean-ui: ## Clean UI.
@(rm -rf ui/build ; echo "UI cleanup done" )
@ -82,23 +86,36 @@ clean-agent: ## Clean agent.
clean-cli: ## Clean CLI.
@(cd cli; make clean ; echo "CLI cleanup done" )
clean-extensions: ## Clean extensions
@(rm -rf tap/extensions/*.so ; echo "Extensions cleanup done" )
clean-docker:
clean-docker: ## Run clean docker
@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
extensions-debug:
devops/build_extensions_debug.sh
clean-bpf:
@(rm $(BPF_O_FILES) ; echo "bpf cleanup done" )
extensions:
devops/build_extensions.sh
test-lint: ## Run lint on all modules
cd agent && golangci-lint run
cd shared && golangci-lint run
cd tap && golangci-lint run
cd cli && golangci-lint run
cd acceptanceTests && golangci-lint run
cd tap/api && golangci-lint run
cd tap/dbgctl && golangci-lint run
cd tap/extensions/ && for D in */; do cd $$D && golangci-lint run && cd ..; done
test-cli:
test-cli: ## Run cli tests
@echo "running cli tests"; cd cli && $(MAKE) test
test-agent:
test-agent: ## Run agent tests
@echo "running agent tests"; cd agent && $(MAKE) test
acceptance-test:
test-shared: ## Run shared tests
@echo "running shared tests"; cd shared && $(MAKE) test
test-extensions: ## Run extensions tests
@echo "running http tests"; cd tap/extensions/http && $(MAKE) test
@echo "running redis tests"; cd tap/extensions/redis && $(MAKE) test
@echo "running kafka tests"; cd tap/extensions/kafka && $(MAKE) test
@echo "running amqp tests"; cd tap/extensions/amqp && $(MAKE) test
acceptance-test: ## Run acceptance tests
@echo "running acceptance tests"; cd acceptanceTests && $(MAKE) test

195
README.md
View File

@ -1,5 +1,23 @@
![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
<p align="center">
<a href="https://github.com/up9inc/mizu/blob/main/LICENSE">
<img alt="GitHub License" src="https://img.shields.io/github/license/up9inc/mizu?logo=GitHub&style=flat-square">
</a>
<a href="https://github.com/up9inc/mizu/releases/latest">
<img alt="GitHub Latest Release" src="https://img.shields.io/github/v/release/up9inc/mizu?logo=GitHub&style=flat-square">
</a>
<a href="https://hub.docker.com/r/up9inc/mizu">
<img alt="Docker pulls" src="https://img.shields.io/docker/pulls/up9inc/mizu?color=%23099cec&logo=Docker&style=flat-square">
</a>
<a href="https://hub.docker.com/r/up9inc/mizu">
<img alt="Image size" src="https://img.shields.io/docker/image-size/up9inc/mizu/latest?logo=Docker&style=flat-square">
</a>
<a href="https://join.slack.com/t/up9/shared_invite/zt-tfjnduli-QzlR8VV4Z1w3YnPIAJfhlQ">
<img alt="Slack" src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social">
</a>
</p>
# The API Traffic Viewer for Kubernetes
A simple-yet-powerful API traffic viewer for Kubernetes enabling you to view all API communication between microservices to help your debug and troubleshoot regressions.
@ -8,178 +26,17 @@ Think TCPDump and Wireshark re-invented for Kubernetes.
![Simple UI](assets/mizu-ui.png)
## Features
## Quickstart and documentation
- Simple and powerful CLI
- Monitoring network traffic in real-time. Supported protocols:
- [HTTP/1.1](https://datatracker.ietf.org/doc/html/rfc2616) (REST, etc.)
- [HTTP/2](https://datatracker.ietf.org/doc/html/rfc7540) (gRPC)
- [AMQP](https://www.rabbitmq.com/amqp-0-9-1-reference.html) (RabbitMQ, Apache Qpid, etc.)
- [Apache Kafka](https://kafka.apache.org/protocol)
- [Redis](https://redis.io/topics/protocol)
- Works with Kubernetes APIs. No installation or code instrumentation
- Rich filtering
You can run Mizu on any Kubernetes cluster (version of 1.16.0 or higher) in a matter of seconds. See the [Mizu Getting Started Guide](https://getmizu.io/docs/) for how.
## Requirements
For more comprehensive documentation, start with the [docs](https://getmizu.io/docs/mizu/mizu-cli).
A Kubernetes server version of 1.16.0 or higher is required.
## Working in this repo
## Download
We ❤️ pull requests! See [CONTRIBUTING.md](docs/CONTRIBUTING.md) for info on contributing changes. <br />
In the wiki you can find an intorduction to [mizu components](https://github.com/up9inc/mizu/wiki/Introduction-to-Mizu), and [development workflows](https://github.com/up9inc/mizu/wiki/Development-Workflows).
Download Mizu for your platform and operating system
## Code of Conduct
### Latest Stable Release
* for MacOS - Intel
```
curl -Lo mizu \
https://github.com/up9inc/mizu/releases/latest/download/mizu_darwin_amd64 \
&& chmod 755 mizu
```
* for Linux - Intel 64bit
```
curl -Lo mizu \
https://github.com/up9inc/mizu/releases/latest/download/mizu_linux_amd64 \
&& chmod 755 mizu
```
SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page
### Development (unstable) Build
Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page
## How to Run
1. Find pods you'd like to tap to in your Kubernetes cluster
2. Run `mizu tap` or `mizu tap PODNAME`
3. Open browser on `http://localhost:8899` **or** as instructed in the CLI
4. Watch the API traffic flowing
5. Type ^C to stop
## Examples
Run `mizu help` for usage options
To tap all pods in current namespace -
```
$ kubectl get pods
NAME READY STATUS RESTARTS AGE
carts-66c77f5fbb-fq65r 2/2 Running 0 20m
catalogue-5f4cb7cf5-7zrmn 2/2 Running 0 20m
front-end-649fc5fd6-kqbtn 2/2 Running 0 20m
..
$ mizu tap
+carts-66c77f5fbb-fq65r
+catalogue-5f4cb7cf5-7zrmn
+front-end-649fc5fd6-kqbtn
Web interface is now available at http://localhost:8899
^C
```
### To tap specific pod
```bash
$ kubectl get pods
NAME READY STATUS RESTARTS AGE
front-end-649fc5fd6-kqbtn 2/2 Running 0 7m
..
$ mizu tap front-end-649fc5fd6-kqbtn
+front-end-649fc5fd6-kqbtn
Web interface is now available at http://localhost:8899
^C
```
### To tap multiple pods using regex
```bash
$ kubectl get pods
NAME READY STATUS RESTARTS AGE
carts-66c77f5fbb-fq65r 2/2 Running 0 20m
catalogue-5f4cb7cf5-7zrmn 2/2 Running 0 20m
front-end-649fc5fd6-kqbtn 2/2 Running 0 20m
..
$ mizu tap "^ca.*"
+carts-66c77f5fbb-fq65r
+catalogue-5f4cb7cf5-7zrmn
Web interface is now available at http://localhost:8899
^C
```
## Configuration
Mizu can optionally work with a config file that can be provided as a CLI argument (using `--set config-path=<PATH>`) or if not provided, will be stored at ${HOME}/.mizu/config.yaml
In case of partial configuration defined, all other fields will be used with defaults <br />
You can always override the defaults or config file with CLI flags
To get the default config params run `mizu config` <br />
To generate a new config file with default values use `mizu config -r`
## Advanced Usage
### Kubeconfig
It is possible to change the kubeconfig path using `KUBECONFIG` environment variable or the command like flag
with `--set kube-config-path=<PATH>`. </br >
If both are not set - Mizu assumes that configuration is at `${HOME}/.kube/config`
### Namespace-Restricted Mode
Some users have permission to only manage resources in one particular namespace assigned to them
By default `mizu tap` creates a new namespace `mizu` for all of its Kubernetes resources. In order to instead install
Mizu in an existing namespace, set the `mizu-resources-namespace` config option
If `mizu-resources-namespace` is set to a value other than the default `mizu`, Mizu will operate in a
Namespace-Restricted mode. It will only tap pods in `mizu-resources-namespace`. This way Mizu only requires permissions
to the namespace set by `mizu-resources-namespace`. The user must set the tapped namespace to the same namespace by
using the `--namespace` flag or by setting `tap.namespaces` in the config file
Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior
For detailed list of k8s permissions see [PERMISSIONS](docs/PERMISSIONS.md) document
### User agent filtering
User-agent filtering (like health checks) - can be configured using command-line options:
```shell
$ mizu tap "^ca.*" --set tap.ignored-user-agents=kube-probe --set tap.ignored-user-agents=prometheus
+carts-66c77f5fbb-fq65r
+catalogue-5f4cb7cf5-7zrmn
Web interface is now available at http://localhost:8899
^C
```
Any request that contains `User-Agent` header with one of the specified values (`kube-probe` or `prometheus`) will not be captured
### Traffic validation rules
This feature allows you to define set of simple rules, and test the traffic against them.
Such validation may test response for specific JSON fields, headers, etc.
Please see [TRAFFIC RULES](docs/POLICY_RULES.md) page for more details and syntax.
### OpenAPI Specification (OAS) Contract Monitoring
An OAS/Swagger file can contain schemas under `parameters` and `responses` fields. With `--contract catalogue.yaml`
CLI option, you can pass your API description to Mizu and the traffic will automatically be validated
against the contracts.
Please see [CONTRACT MONITORING](docs/CONTRACT_MONITORING.md) page for more details and syntax.
### Configure proxy host
By default, mizu will be accessible via local host: 'http://localhost:8899', it is possible to change the host, for
instance, to '0.0.0.0' which can grant access via machine IP address. This setting can be changed via command line
flag `--set tap.proxy-host=<value>` or via config file:
tap proxy-host: 0.0.0.0 and when changed it will support accessing by IP
### Install Mizu standalone
Mizu can be run detached from the cli using the install command: `mizu install`. This type of mizu instance will run
indefinitely in the cluster.
For more information please refer to [INSTALL STANDALONE](docs/INSTALL_STANDALONE.md)
This project is for everyone. We ask that our users and contributors take a few minutes to review our [Code of Conduct](docs/CODE_OF_CONDUCT.md).

6
acceptanceTests/.snyk Normal file
View File

@ -0,0 +1,6 @@
# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
version: v1.14.0
ignore:
SNYK-GOLANG-GITHUBCOMPKGSFTP-569475:
- '*':
reason: None Given

View File

@ -1,2 +1,3 @@
test: ## Run acceptance tests.
@npm install cypress@10.0.1 -y
@go test ./... -timeout 1h -v

View File

@ -23,19 +23,19 @@ func TestConfigRegenerate(t *testing.T) {
t.Skip("ignored acceptance test")
}
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
configPath, configPathErr := getConfigPath()
configPath, configPathErr := GetConfigPath()
if configPathErr != nil {
t.Errorf("failed to get config path, err: %v", cliPathErr)
return
}
configCmdArgs := getDefaultConfigCommandArgs()
configCmdArgs := GetDefaultConfigCommandArgs()
configCmdArgs = append(configCmdArgs, "-r")
@ -74,13 +74,13 @@ func TestConfigGuiPort(t *testing.T) {
for _, guiPort := range tests {
t.Run(fmt.Sprintf("%d", guiPort), func(t *testing.T) {
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
configPath, configPathErr := getConfigPath()
configPath, configPathErr := GetConfigPath()
if configPathErr != nil {
t.Errorf("failed to get config path, err: %v", cliPathErr)
return
@ -100,16 +100,16 @@ func TestConfigGuiPort(t *testing.T) {
return
}
tapCmdArgs := getDefaultTapCommandArgs()
tapCmdArgs := GetDefaultTapCommandArgs()
tapNamespace := getDefaultTapNamespace()
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
tapCmd := exec.Command(cliPath, tapCmdArgs...)
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := cleanupCommand(tapCmd); err != nil {
if err := CleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
@ -123,9 +123,9 @@ func TestConfigGuiPort(t *testing.T) {
return
}
apiServerUrl := getApiServerUrl(guiPort)
apiServerUrl := GetApiServerUrl(guiPort)
if err := waitTapPodsReady(apiServerUrl); err != nil {
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
@ -147,13 +147,13 @@ func TestConfigSetGuiPort(t *testing.T) {
for _, guiPortStruct := range tests {
t.Run(fmt.Sprintf("%d", guiPortStruct.SetGuiPort), func(t *testing.T) {
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
configPath, configPathErr := getConfigPath()
configPath, configPathErr := GetConfigPath()
if configPathErr != nil {
t.Errorf("failed to get config path, err: %v", cliPathErr)
return
@ -173,9 +173,9 @@ func TestConfigSetGuiPort(t *testing.T) {
return
}
tapCmdArgs := getDefaultTapCommandArgs()
tapCmdArgs := GetDefaultTapCommandArgs()
tapNamespace := getDefaultTapNamespace()
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
tapCmdArgs = append(tapCmdArgs, "--set", fmt.Sprintf("tap.gui-port=%v", guiPortStruct.SetGuiPort))
@ -184,7 +184,7 @@ func TestConfigSetGuiPort(t *testing.T) {
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := cleanupCommand(tapCmd); err != nil {
if err := CleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
@ -198,9 +198,9 @@ func TestConfigSetGuiPort(t *testing.T) {
return
}
apiServerUrl := getApiServerUrl(guiPortStruct.SetGuiPort)
apiServerUrl := GetApiServerUrl(guiPortStruct.SetGuiPort)
if err := waitTapPodsReady(apiServerUrl); err != nil {
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
@ -222,13 +222,13 @@ func TestConfigFlagGuiPort(t *testing.T) {
for _, guiPortStruct := range tests {
t.Run(fmt.Sprintf("%d", guiPortStruct.FlagGuiPort), func(t *testing.T) {
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
configPath, configPathErr := getConfigPath()
configPath, configPathErr := GetConfigPath()
if configPathErr != nil {
t.Errorf("failed to get config path, err: %v", cliPathErr)
return
@ -248,9 +248,9 @@ func TestConfigFlagGuiPort(t *testing.T) {
return
}
tapCmdArgs := getDefaultTapCommandArgs()
tapCmdArgs := GetDefaultTapCommandArgs()
tapNamespace := getDefaultTapNamespace()
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
tapCmdArgs = append(tapCmdArgs, "-p", fmt.Sprintf("%v", guiPortStruct.FlagGuiPort))
@ -259,7 +259,7 @@ func TestConfigFlagGuiPort(t *testing.T) {
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := cleanupCommand(tapCmd); err != nil {
if err := CleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
@ -273,9 +273,9 @@ func TestConfigFlagGuiPort(t *testing.T) {
return
}
apiServerUrl := getApiServerUrl(guiPortStruct.FlagGuiPort)
apiServerUrl := GetApiServerUrl(guiPortStruct.FlagGuiPort)
if err := waitTapPodsReady(apiServerUrl); err != nil {
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}

37
acceptanceTests/create_user.sh Executable file
View File

@ -0,0 +1,37 @@
#!/bin/bash
# Create a user in Minikube cluster "minikube"
# Create context for user
# Usage:
# ./create_user.sh <username>
set -e
NEW_USERNAME=$1
CERT_DIR="${HOME}/certs"
KEY_FILE="${CERT_DIR}/${NEW_USERNAME}.key"
CRT_FILE="${CERT_DIR}/${NEW_USERNAME}.crt"
MINIKUBE_KEY_FILE="${HOME}/.minikube/ca.key"
MINIKUBE_CRT_FILE="${HOME}/.minikube/ca.crt"
DAYS=1
echo "Creating user and context for username \"${NEW_USERNAME}\" in Minikube cluster"
if ! command -v openssl &> /dev/null
then
echo "Installing openssl"
sudo apt-get update
sudo apt-get install openssl
fi
echo "Creating certificate for user \"${NEW_USERNAME}\""
mkdir -p ${CERT_DIR}
echo "Generating key \"${KEY_FILE}\""
openssl genrsa -out "${KEY_FILE}" 2048
echo "Generating crt \"${CRT_FILE}\""
openssl req -new -key "${KEY_FILE}" -out "${CRT_FILE}" -subj "/CN=${NEW_USERNAME}/O=group1"
openssl x509 -req -in "${CRT_FILE}" -CA "${MINIKUBE_CRT_FILE}" -CAkey "${MINIKUBE_KEY_FILE}" -CAcreateserial -out "${CRT_FILE}" -days $DAYS
echo "Creating context for user \"${NEW_USERNAME}\""
kubectl config set-credentials "${NEW_USERNAME}" --client-certificate="${CRT_FILE}" --client-key="${KEY_FILE}"
kubectl config set-context "${NEW_USERNAME}" --cluster=minikube --user="${NEW_USERNAME}"

View File

@ -0,0 +1,30 @@
const { defineConfig } = require('cypress')
module.exports = defineConfig({
watchForFileChanges: false,
viewportWidth: 1920,
viewportHeight: 1080,
video: false,
screenshotOnRunFailure: false,
defaultCommandTimeout: 6000,
env: {
testUrl: 'http://localhost:8899/',
redactHeaderContent: 'User-Header[REDACTED]',
redactBodyContent: '{ "User": "[REDACTED]" }',
greenFilterColor: 'rgb(210, 250, 210)',
redFilterColor: 'rgb(250, 214, 220)',
bodyJsonClass: '.hljs',
mizuWidth: 1920,
normalMizuHeight: 1080,
hugeMizuHeight: 3500,
},
e2e: {
// We've imported your old cypress plugins here.
// You may want to clean this up later by importing these.
// setupNodeEvents(on, config) {
// return require('./cypress/plugins/index.js')(on, config)
// },
specPattern: 'cypress/e2e/tests/*.js',
supportFile: false
},
})

View File

@ -1,14 +0,0 @@
{
"watchForFileChanges":false,
"viewportWidth": 1920,
"viewportHeight": 1080,
"video": false,
"screenshotOnRunFailure": false,
"testFiles":
["tests/GuiPort.js",
"tests/MultipleNamespaces.js",
"tests/Regex.js"],
"env": {
"testUrl": "http://localhost:8899/"
}
}

View File

@ -1,8 +1,7 @@
const columns = {podName : 1, namespace : 2, tapping : 3};
const greenStatusImageSrc = '/static/media/success.662997eb.svg';
function getDomPathInStatusBar(line, column) {
return `.expandedStatusBar > :nth-child(2) > > :nth-child(2) > :nth-child(${line}) > :nth-child(${column})`;
return `[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) > :nth-child(${line}) > :nth-child(${column})`;
}
export function checkLine(line, expectedValues) {
@ -12,14 +11,14 @@ export function checkLine(line, expectedValues) {
cy.get(getDomPathInStatusBar(line, columns.namespace)).invoke('text').then(namespaceValue => {
expect(namespaceValue).to.equal(expectedValues.namespace);
cy.get(getDomPathInStatusBar(line, columns.tapping)).children().should('have.attr', 'src', greenStatusImageSrc);
cy.get(getDomPathInStatusBar(line, columns.tapping)).children().should('have.attr', 'src').and("match", /success.*\.svg/);
});
});
}
export function findLineAndCheck(expectedValues) {
cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) > > :nth-child(1)').then(pods => {
cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) > > :nth-child(2)').then(namespaces => {
cy.get('[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) > > :nth-child(1)').then(pods => {
cy.get('[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) > > :nth-child(2)').then(namespaces => {
// organizing namespaces array
const podObjectsArray = Object.values(pods ?? {});
const namespacesObjectsArray = Object.values(namespaces ?? {});

View File

@ -0,0 +1,181 @@
export const valueTabs = {
response: 'RESPONSE',
request: 'REQUEST',
none: null
}
export function isValueExistsInElement(shouldInclude, content, domPathToContainer){
it(`should ${shouldInclude ? '' : 'not'} include '${content}'`, function () {
cy.get(domPathToContainer).then(htmlText => {
const allTextString = htmlText.text();
if (allTextString.includes(content) !== shouldInclude)
throw new Error(`One of the containers part contains ${content}`)
});
});
}
export function resizeToHugeMizu() {
cy.viewport(Cypress.env('mizuWidth'), Cypress.env('hugeMizuHeight'));
}
export function resizeToNormalMizu() {
cy.viewport(Cypress.env('mizuWidth'), Cypress.env('normalMizuHeight'));
}
export function verifyMinimumEntries() {
const entriesSent = Cypress.env('entriesCount');
const minimumEntries = Math.round((0.75 * entriesSent));
it(`Making sure that mizu shows at least ${minimumEntries} entries`, function () {
cy.get('#total-entries').then(number => {
const getNum = () => {
return parseInt(number.text());
};
cy.wrap({num: getNum}).invoke('num').should('be.gt', minimumEntries);
});
});
}
export function leftTextCheck(entryId, path, expectedText) {
cy.get(`#list #entry-${entryId} ${path}`).invoke('text').should('eq', expectedText);
}
export function leftOnHoverCheck(entryId, path, filterName) {
cy.get(`#list #entry-${entryId} ${path}`).trigger('mouseover');
cy.get(`#list #entry-${entryId} [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(filterName));
}
export function rightTextCheck(path, expectedText) {
cy.get(`#rightSideContainer ${path}`).should('have.text', expectedText);
}
export function rightOnHoverCheck(path, expectedText) {
cy.get(`#rightSideContainer ${path}`).trigger('mouseover');
cy.get(`#rightSideContainer [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(expectedText));
}
export function checkFilterByMethod(funcDict) {
const {protocol, method, methodQuery, summary, summaryQuery, numberOfRecords} = funcDict;
const summaryDict = getSummaryDict(summary, summaryQuery);
const methodDict = getMethodDict(method, methodQuery);
const protocolDict = getProtocolDict(protocol.name, protocol.text);
it(`Testing the method: ${method}`, function () {
// applying filter
cy.get('.w-tc-editor-text').clear().type(methodQuery);
cy.get('[type="submit"]').click();
cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
waitForFetch(numberOfRecords);
pauseStream();
cy.get(`#list [id^=entry]`).then(elements => {
const listElmWithIdAttr = Object.values(elements);
let doneCheckOnFirst = false;
cy.get('#entries-length').invoke('text').then(len => {
listElmWithIdAttr.forEach(entry => {
if (entry?.id && entry.id.match(RegExp(/entry-(\d{24})$/gm))) {
const entryId = getEntryId(entry.id);
leftTextCheck(entryId, methodDict.pathLeft, methodDict.expectedText);
leftTextCheck(entryId, protocolDict.pathLeft, protocolDict.expectedTextLeft);
if (summaryDict)
leftTextCheck(entryId, summaryDict.pathLeft, summaryDict.expectedText);
if (!doneCheckOnFirst) {
deepCheck(funcDict, protocolDict, methodDict, entry);
doneCheckOnFirst = true;
}
}
});
});
});
});
}
export const refreshWaitTimeout = 10000;
export function waitForFetch(gt) {
cy.get('#entries-length', {timeout: refreshWaitTimeout}).should((el) => {
expect(parseInt(el.text().trim(), 10)).to.be.greaterThan(gt);
});
}
export function pauseStream() {
cy.get('#pause-icon').click();
cy.get('#pause-icon').should('not.be.visible');
}
export function getEntryId(id) {
// take the second part from the string (entry-<ID>)
return id.split('-')[1];
}
function deepCheck(generalDict, protocolDict, methodDict, entry) {
const entryId = getEntryId(entry.id);
const {summary, value} = generalDict;
const summaryDict = getSummaryDict(summary);
leftOnHoverCheck(entryId, methodDict.pathLeft, methodDict.expectedOnHover);
leftOnHoverCheck(entryId, protocolDict.pathLeft, protocolDict.expectedOnHover);
if (summaryDict)
leftOnHoverCheck(entryId, summaryDict.pathLeft, summaryDict.expectedOnHover);
cy.get(`#${entry.id}`).click();
rightTextCheck(methodDict.pathRight, methodDict.expectedText);
rightTextCheck(protocolDict.pathRight, protocolDict.expectedTextRight);
if (summaryDict)
rightTextCheck(summaryDict.pathRight, summaryDict.expectedText);
rightOnHoverCheck(methodDict.pathRight, methodDict.expectedOnHover);
rightOnHoverCheck(protocolDict.pathRight, protocolDict.expectedOnHover);
if (summaryDict)
rightOnHoverCheck(summaryDict.pathRight, summaryDict.expectedOnHover);
if (value) {
if (value.tab === valueTabs.response)
// temporary fix, change to some "data-cy" attribute,
// this will fix the issue that happen because we have "response:" in the header of the right side
cy.get('#rightSideContainer > :nth-child(3)').contains('Response').click();
cy.get(Cypress.env('bodyJsonClass')).then(text => {
expect(text.text()).to.match(value.regex)
});
}
}
function getSummaryDict(value, query) {
if (value) {
return {
pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
expectedText: value,
expectedOnHover: query
};
}
else {
return null;
}
}
function getMethodDict(value, query) {
return {
pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
expectedText: value,
expectedOnHover: query
};
}
function getProtocolDict(protocol, protocolText) {
return {
pathLeft: '> :nth-child(1) > :nth-child(1)',
pathRight: '> :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(1)',
expectedTextLeft: protocol.toUpperCase(),
expectedTextRight: protocolText,
expectedOnHover: protocol.toLowerCase()
};
}

View File

@ -0,0 +1,13 @@
import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
it('check', function () {
const podName = Cypress.env('name'), namespace = Cypress.env('namespace');
const port = Cypress.env('port');
cy.intercept('GET', `http://localhost:${port}/status/tap`).as('statusTap');
cy.visit(`http://localhost:${port}`);
cy.wait('@statusTap').its('response.statusCode').should('match', /^2\d{2}/);
cy.get(`[data-cy="expandedStatusBar"]`).trigger('mouseover',{force: true});
findLineAndCheck(getExpectedDetailsDict(podName, namespace));
});

View File

@ -0,0 +1,25 @@
import {
isValueExistsInElement,
resizeToHugeMizu,
} from "../testHelpers/TrafficHelper";
it('Loading Mizu', function () {
cy.visit(Cypress.env('testUrl'));
});
checkEntries();
function checkEntries() {
it('checking all entries', function () {
cy.get('#entries-length').should('not.have.text', '0').then(() => {
resizeToHugeMizu();
cy.get('#list [id^=entry]').each(entryElement => {
entryElement.click();
cy.get('#tbody-Headers').should('be.visible');
isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
});
});
});
}

View File

@ -1,8 +1,8 @@
import {findLineAndCheck, getExpectedDetailsDict} from '../page_objects/StatusBar';
import {findLineAndCheck, getExpectedDetailsDict} from '../testHelpers/StatusBarHelper';
it('opening', function () {
cy.visit(Cypress.env('testUrl'));
cy.get('.podsCount').trigger('mouseover');
cy.get(`[data-cy="podsCountText"]`).trigger('mouseover');
});
[1, 2, 3].map(doItFunc);
@ -15,4 +15,3 @@ function doItFunc(number) {
findLineAndCheck(getExpectedDetailsDict(podName, namespace));
});
}

View File

@ -0,0 +1,8 @@
import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
it('Loading Mizu', function () {
cy.visit(Cypress.env('testUrl'));
});
isValueExistsInElement(false, Cypress.env('redactHeaderContent'), '#tbody-Headers');
isValueExistsInElement(false, Cypress.env('redactBodyContent'), Cypress.env('bodyJsonClass'));

View File

@ -0,0 +1,68 @@
import {checkFilterByMethod, valueTabs,} from "../testHelpers/TrafficHelper";
it('opening mizu', function () {
cy.visit(Cypress.env('testUrl'));
});
const rabbitProtocolDetails = {name: 'AMQP', text: 'Advanced Message Queuing Protocol 0-9-1'};
const numberOfRecords = 5;
checkFilterByMethod({
protocol: rabbitProtocolDetails,
method: 'exchange declare',
methodQuery: 'request.method == "exchange declare"',
summary: 'exchange',
summaryQuery: 'request.exchange == "exchange"',
numberOfRecords: numberOfRecords,
value: null
});
checkFilterByMethod({
protocol: rabbitProtocolDetails,
method: 'queue declare',
methodQuery: 'request.method == "queue declare"',
summary: 'queue',
summaryQuery: 'request.queue == "queue"',
numberOfRecords: numberOfRecords,
value: null
});
checkFilterByMethod({
protocol: rabbitProtocolDetails,
method: 'queue bind',
methodQuery: 'request.method == "queue bind"',
summary: 'queue',
summaryQuery: 'request.queue == "queue"',
numberOfRecords: numberOfRecords,
value: null
});
checkFilterByMethod({
protocol: rabbitProtocolDetails,
method: 'basic publish',
methodQuery: 'request.method == "basic publish"',
summary: 'exchange',
summaryQuery: 'request.exchange == "exchange"',
numberOfRecords: numberOfRecords,
value: {tab: valueTabs.request, regex: /^message$/mg}
});
checkFilterByMethod({
protocol: rabbitProtocolDetails,
method: 'basic consume',
methodQuery: 'request.method == "basic consume"',
summary: 'queue',
summaryQuery: 'request.queue == "queue"',
numberOfRecords: numberOfRecords,
value: null
});
checkFilterByMethod({
protocol: rabbitProtocolDetails,
method: 'basic deliver',
methodQuery: 'request.method == "basic deliver"',
summary: 'exchange',
summaryQuery: 'request.queue == "exchange"',
numberOfRecords: numberOfRecords,
value: {tab: valueTabs.request, regex: /^message$/mg}
});

View File

@ -0,0 +1,8 @@
import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
it('Loading Mizu', function () {
cy.visit(Cypress.env('testUrl'));
});
isValueExistsInElement(true, Cypress.env('redactHeaderContent'), '#tbody-Headers');
isValueExistsInElement(true, Cypress.env('redactBodyContent'), Cypress.env('bodyJsonClass'));

View File

@ -0,0 +1,58 @@
import {checkFilterByMethod, valueTabs,} from "../testHelpers/TrafficHelper";
it('opening mizu', function () {
cy.visit(Cypress.env('testUrl'));
});
const redisProtocolDetails = {name: 'redis', text: 'Redis Serialization Protocol'};
const numberOfRecords = 5;
checkFilterByMethod({
protocol: redisProtocolDetails,
method: 'PING',
methodQuery: 'request.command == "PING"',
summary: null,
summaryQuery: '',
numberOfRecords: numberOfRecords,
value: null
})
checkFilterByMethod({
protocol: redisProtocolDetails,
method: 'SET',
methodQuery: 'request.command == "SET"',
summary: 'key',
summaryQuery: 'request.key == "key"',
numberOfRecords: numberOfRecords,
value: {tab: valueTabs.request, regex: /^\[value, keepttl]$/mg}
})
checkFilterByMethod({
protocol: redisProtocolDetails,
method: 'EXISTS',
methodQuery: 'request.command == "EXISTS"',
summary: 'key',
summaryQuery: 'request.key == "key"',
numberOfRecords: numberOfRecords,
value: {tab: valueTabs.response, regex: /^1$/mg}
})
checkFilterByMethod({
protocol: redisProtocolDetails,
method: 'GET',
methodQuery: 'request.command == "GET"',
summary: 'key',
summaryQuery: 'request.key == "key"',
numberOfRecords: numberOfRecords,
value: {tab: valueTabs.response, regex: /^value$/mg}
})
checkFilterByMethod({
protocol: redisProtocolDetails,
method: 'DEL',
methodQuery: 'request.command == "DEL"',
summary: 'key',
summaryQuery: 'request.key == "key"',
numberOfRecords: numberOfRecords,
value: {tab: valueTabs.response, regex: /^1$|^0$/mg}
})

View File

@ -0,0 +1,11 @@
import {getExpectedDetailsDict, checkLine} from '../testHelpers/StatusBarHelper';
it('opening', function () {
cy.visit(Cypress.env('testUrl'));
cy.get(`[data-cy="podsCountText"]`).trigger('mouseover');
cy.get('[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) >').should('have.length', 1); // one line
checkLine(1, getExpectedDetailsDict(Cypress.env('name'), Cypress.env('namespace')));
});

View File

@ -0,0 +1,351 @@
import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
import {
getEntryId,
leftOnHoverCheck,
leftTextCheck,
resizeToHugeMizu,
resizeToNormalMizu,
rightOnHoverCheck,
rightTextCheck,
verifyMinimumEntries,
refreshWaitTimeout,
waitForFetch,
pauseStream
} from "../testHelpers/TrafficHelper";
const fullParam = Cypress.env('arrayDict'); // "Name:fooNamespace:barName:foo1Namespace:bar1"
const podsArray = fullParam.split('Name:').slice(1); // ["fooNamespace:bar", "foo1Namespace:bar1"]
podsArray.forEach((podStr, index) => {
const podAndNamespaceArr = podStr.split('Namespace:'); // [foo, bar] / [foo1, bar1]
podsArray[index] = getExpectedDetailsDict(podAndNamespaceArr[0], podAndNamespaceArr[1]);
});
it('opening mizu', function () {
cy.visit(Cypress.env('testUrl'));
});
verifyMinimumEntries();
it('top bar check', function () {
cy.get(`[data-cy="podsCountText"]`).trigger('mouseover');
podsArray.map(findLineAndCheck);
cy.reload();
});
it('filtering guide check', function () {
cy.reload();
cy.get('[title="Open Filtering Guide (Cheatsheet)"]').click();
cy.get('#modal-modal-title').should('be.visible');
cy.get('[lang="en"]').click(0, 0);
cy.get('#modal-modal-title').should('not.exist');
});
it('right side sanity test', function () {
cy.get('#entryDetailedTitleElapsedTime').then(timeInMs => {
const time = timeInMs.text();
if (time < '0ms') {
throw new Error(`The time in the top line cannot be negative ${time}`);
}
});
// temporary fix, change to some "data-cy" attribute,
// this will fix the issue that happen because we have "response:" in the header of the right side
cy.get('#rightSideContainer > :nth-child(3)').contains('Response').click();
cy.get('#rightSideContainer [title="Status Code"]').then(status => {
const statusCode = status.text();
cy.contains('Status').parent().next().then(statusInDetails => {
const statusCodeInDetails = statusInDetails.text();
expect(statusCode).to.equal(statusCodeInDetails, 'The status code in the top line should match the status code in details');
});
});
});
checkIllegalFilter('invalid filter');
checkFilter({
filter: 'http',
leftSidePath: '> :nth-child(1) > :nth-child(1)',
leftSideExpectedText: 'HTTP',
rightSidePath: '[title=HTTP]',
rightSideExpectedText: 'Hypertext Transfer Protocol -- HTTP/1.1',
applyByCtrlEnter: true,
numberOfRecords: 20,
});
checkFilter({
filter: 'response.status == 200',
leftSidePath: '[title="Status Code"]',
leftSideExpectedText: '200',
rightSidePath: '> :nth-child(2) [title="Status Code"]',
rightSideExpectedText: '200',
applyByCtrlEnter: false,
numberOfRecords: 20
});
if (Cypress.env('shouldCheckSrcAndDest')) {
serviceMapCheck();
checkFilter({
filter: 'src.name == ""',
leftSidePath: '[title="Source Name"]',
leftSideExpectedText: '[Unresolved]',
rightSidePath: '> :nth-child(2) [title="Source Name"]',
rightSideExpectedText: '[Unresolved]',
applyByCtrlEnter: false,
numberOfRecords: 20
});
checkFilter({
filter: `dst.name == "httpbin.mizu-tests"`,
leftSidePath: '> :nth-child(3) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
leftSideExpectedText: 'httpbin.mizu-tests',
rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
rightSideExpectedText: 'httpbin.mizu-tests',
applyByCtrlEnter: false,
numberOfRecords: 20
});
}
checkFilter({
filter: 'request.method == "GET"',
leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
leftSideExpectedText: 'GET',
rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
rightSideExpectedText: 'GET',
applyByCtrlEnter: true,
numberOfRecords: 20
});
checkFilter({
filter: 'request.path == "/get"',
leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
leftSideExpectedText: '/get',
rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
rightSideExpectedText: '/get',
applyByCtrlEnter: false,
numberOfRecords: 20
});
checkFilter({
filter: 'src.ip == "127.0.0.1"',
leftSidePath: '[title="Source IP"]',
leftSideExpectedText: '127.0.0.1',
rightSidePath: '> :nth-child(2) [title="Source IP"]',
rightSideExpectedText: '127.0.0.1',
applyByCtrlEnter: false,
numberOfRecords: 20
});
checkFilterNoResults('request.method == "POST"');
function checkFilterNoResults(filterName) {
it(`checking the filter: ${filterName}. Expecting no results`, function () {
cy.get('#total-entries').then(number => {
const totalEntries = number.text();
// applying the filter
cy.get('.w-tc-editor-text').type(filterName);
cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
cy.get('[type="submit"]').click();
// waiting for the entries number to load
cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
// the DOM should show 0 entries
cy.get('#entries-length').should('have.text', '0');
cy.get('[title="Fetch old records"]').click();
cy.get('#noMoreDataTop', {timeout: refreshWaitTimeout}).should('be.visible');
cy.get('#entries-length').should('have.text', '0'); // after loading all entries there should still be 0 entries
// reloading then waiting for the entries number to load
cy.reload();
cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
});
});
}
function checkIllegalFilter(illegalFilterName) {
it(`should show red search bar with the input: ${illegalFilterName}`, function () {
cy.reload();
cy.get('#total-entries').then(number => {
const totalEntries = number.text();
cy.get('.w-tc-editor-text').type(illegalFilterName);
cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('redFilterColor'));
cy.get('[type="submit"]').click();
cy.get('[role="alert"]').should('be.visible');
cy.get('.w-tc-editor-text').clear();
// reloading then waiting for the entries number to load
cy.reload();
cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
});
});
}
function checkFilter(filterDetails) {
const {
filter,
leftSidePath,
rightSidePath,
rightSideExpectedText,
leftSideExpectedText,
applyByCtrlEnter,
numberOfRecords
} = filterDetails;
const entriesForDeeperCheck = 5;
it(`checking the filter: ${filter}`, function () {
cy.get('.w-tc-editor-text').clear();
// applying the filter with alt+enter or with the button
cy.get('.w-tc-editor-text').type(`${filter}${applyByCtrlEnter ? '{ctrl+enter}' : ''}`);
cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
if (!applyByCtrlEnter)
cy.get('[type="submit"]').click();
waitForFetch(numberOfRecords);
pauseStream();
cy.get(`#list [id^=entry]`).last().then(elem => {
const element = elem[0];
const entryId = getEntryId(element.id);
// only one entry in DOM after filtering, checking all checks on it
leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
leftOnHoverCheck(entryId, leftSidePath, filter);
rightTextCheck(rightSidePath, rightSideExpectedText);
rightOnHoverCheck(rightSidePath, filter);
checkRightSideResponseBody();
});
resizeToHugeMizu();
// checking only 'leftTextCheck' on all entries because the rest of the checks require more time
cy.get(`#list [id^=entry]`).each(elem => {
const element = elem[0];
let entryId = getEntryId(element.id);
leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
});
// making the other 3 checks on the first X entries (longer time for each check)
deeperCheck(leftSidePath, rightSidePath, filter, rightSideExpectedText, entriesForDeeperCheck);
// reloading then waiting for the entries number to load
resizeToNormalMizu();
cy.reload();
waitForFetch(numberOfRecords);
pauseStream();
});
}
function deeperCheck(leftSidePath, rightSidePath, filterName, rightSideExpectedText, entriesNumToCheck) {
cy.get(`#list [id^=entry]`).each((element, index) => {
if (index < entriesNumToCheck) {
const entryId = getEntryId(element[0].id);
leftOnHoverCheck(entryId, leftSidePath, filterName);
cy.get(`#list #entry-${entryId}`).click();
rightTextCheck(rightSidePath, rightSideExpectedText);
rightOnHoverCheck(rightSidePath, filterName);
}
});
}
function checkRightSideResponseBody() {
// temporary fix, change to some "data-cy" attribute,
// this will fix the issue that happen because we have "response:" in the header of the right side
cy.get('#rightSideContainer > :nth-child(3)').contains('Response').click();
clickCheckbox('Decode Base64');
cy.get(`${Cypress.env('bodyJsonClass')}`).then(value => {
const encodedBody = value.text();
const decodedBody = atob(encodedBody);
const responseBody = JSON.parse(decodedBody);
const expectedJsonBody = {
args: RegExp({}),
url: RegExp('http://.*/get'),
headers: {
"User-Agent": RegExp('client'),
"Accept-Encoding": RegExp('gzip'),
"X-Forwarded-Uri": RegExp('/api/v1/namespaces/.*/services/.*/proxy/get')
}
};
const expectedStringInJsonBody = RegExp('/api/v1/namespaces/.*/services/.*/proxy/get');
expect(responseBody.args).to.match(expectedJsonBody.args);
expect(responseBody.url).to.match(expectedJsonBody.url);
expect(responseBody.headers['User-Agent']).to.match(expectedJsonBody.headers['User-Agent']);
expect(responseBody.headers['Accept-Encoding']).to.match(expectedJsonBody.headers['Accept-Encoding']);
expect(responseBody.headers['X-Forwarded-Uri']).to.match(expectedJsonBody.headers['X-Forwarded-Uri']);
cy.get(`${Cypress.env('bodyJsonClass')}`).should('have.text', encodedBody);
cy.get(`[data-cy="lineNumbersCheckBoxInput"]`).should('be.disabled');
clickCheckbox('Decode Base64');
cy.get(`[data-cy="lineNumbersCheckBoxInput"]`).should('not.be.disabled');
cy.get(`${Cypress.env('bodyJsonClass')} > `).its('length').should('be.gt', 1).then(linesNum => {
cy.get(`${Cypress.env('bodyJsonClass')} > >`).its('length').should('be.gt', linesNum).then(jsonItemsNum => {
checkOnlyLineNumberes(jsonItemsNum, expectedStringInJsonBody);
});
});
});
}
function clickCheckbox(type) {
cy.contains(`${type}`).prev().children().click();
}
function checkOnlyLineNumberes(jsonItems, decodedText) {
cy.get(`${Cypress.env('bodyJsonClass')} > >`).should('have.length', jsonItems);
cy.get(`${Cypress.env('bodyJsonClass')} >`).contains(decodedText);
}
function serviceMapCheck() {
it('service map test', function () {
cy.intercept(`${Cypress.env('testUrl')}/servicemap/get`).as('serviceMapRequest');
cy.get('#total-entries').should('not.have.text', '0').then(() => {
cy.get('#total-entries').invoke('text').then(entriesNum => {
cy.get('[alt="service-map"]').click();
cy.wait('@serviceMapRequest').then(({response}) => {
const body = response.body;
const nodeParams = {
destination: 'httpbin.mizu-tests',
source: '127.0.0.1'
};
serviceMapAPICheck(body, parseInt(entriesNum), nodeParams);
cy.reload();
});
});
});
});
}
function serviceMapAPICheck(body, entriesNum, nodeParams) {
const {nodes, edges} = body;
expect(nodes.length).to.equal(Object.keys(nodeParams).length, `Expected nodes count`);
expect(edges.some(edge => edge.source.name === nodeParams.source)).to.be.true;
expect(edges.some(edge => edge.destination.name === nodeParams.destination)).to.be.true;
let count = 0;
edges.forEach(edge => {
count += edge.count;
if (edge.destination.name === nodeParams.destination) {
expect(edge.source.name).to.equal(nodeParams.source);
}
});
expect(count).to.equal(entriesNum);
}

View File

@ -1,8 +0,0 @@
it('check', function () {
cy.visit(`http://localhost:${Cypress.env('port')}/`);
cy.get('.header').should('be.visible');
cy.get('.TrafficPageHeader').should('be.visible');
cy.get('.TrafficPage-ListContainer').should('be.visible');
cy.get('.TrafficPage-Container').should('be.visible');
});

View File

@ -1,11 +0,0 @@
import {getExpectedDetailsDict, checkLine} from '../page_objects/StatusBar';
it('opening', function () {
cy.visit(Cypress.env('testUrl'));
cy.get('.podsCount').trigger('mouseover');
cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) >').should('have.length', 1); // one line
checkLine(1, getExpectedDetailsDict(Cypress.env('name'), Cypress.env('namespace')));
});

View File

@ -0,0 +1,243 @@
package acceptanceTests
import (
"context"
"fmt"
"github.com/go-redis/redis/v8"
amqp "github.com/rabbitmq/amqp091-go"
"os/exec"
"testing"
"time"
)
func TestRedis(t *testing.T) {
if testing.Short() {
t.Skip("ignored acceptance test")
}
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
tapCmdArgs := GetDefaultTapCommandArgs()
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
tapCmd := exec.Command(cliPath, tapCmdArgs...)
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := CleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
})
if err := tapCmd.Start(); err != nil {
t.Errorf("failed to start tap command, err: %v", err)
return
}
apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
ctx := context.Background()
kubernetesProvider, err := NewKubernetesProvider()
if err != nil {
t.Errorf("failed to create k8s provider, err %v", err)
return
}
redisExternalIp, err := kubernetesProvider.GetServiceExternalIp(ctx, DefaultNamespaceName, "redis")
if err != nil {
t.Errorf("failed to get redis external ip, err: %v", err)
return
}
rdb := redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("%v:6379", redisExternalIp),
})
for i := 0; i < DefaultEntriesCount/5; i++ {
requestErr := rdb.Ping(ctx).Err()
if requestErr != nil {
t.Errorf("failed to send redis request, err: %v", requestErr)
return
}
}
for i := 0; i < DefaultEntriesCount/5; i++ {
requestErr := rdb.Set(ctx, "key", "value", -1).Err()
if requestErr != nil {
t.Errorf("failed to send redis request, err: %v", requestErr)
return
}
}
for i := 0; i < DefaultEntriesCount/5; i++ {
requestErr := rdb.Exists(ctx, "key").Err()
if requestErr != nil {
t.Errorf("failed to send redis request, err: %v", requestErr)
return
}
}
for i := 0; i < DefaultEntriesCount/5; i++ {
requestErr := rdb.Get(ctx, "key").Err()
if requestErr != nil {
t.Errorf("failed to send redis request, err: %v", requestErr)
return
}
}
for i := 0; i < DefaultEntriesCount/5; i++ {
requestErr := rdb.Del(ctx, "key").Err()
if requestErr != nil {
t.Errorf("failed to send redis request, err: %v", requestErr)
return
}
}
RunCypressTests(t, "npx cypress run --spec \"cypress/e2e/tests/Redis.js\"")
}
func TestAmqp(t *testing.T) {
t.Skip("ignoredd for now because those tests are not stable")
if testing.Short() {
t.Skip("ignored acceptance test")
}
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
tapCmdArgs := GetDefaultTapCommandArgs()
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
tapCmd := exec.Command(cliPath, tapCmdArgs...)
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := CleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
})
if err := tapCmd.Start(); err != nil {
t.Errorf("failed to start tap command, err: %v", err)
return
}
apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
ctx := context.Background()
kubernetesProvider, err := NewKubernetesProvider()
if err != nil {
t.Errorf("failed to create k8s provider, err %v", err)
return
}
rabbitmqExternalIp, err := kubernetesProvider.GetServiceExternalIp(ctx, DefaultNamespaceName, "rabbitmq")
if err != nil {
t.Errorf("failed to get RabbitMQ external ip, err: %v", err)
return
}
conn, err := amqp.Dial(fmt.Sprintf("amqp://guest:guest@%v:5672/", rabbitmqExternalIp))
if err != nil {
t.Errorf("failed to connect to RabbitMQ, err: %v", err)
return
}
defer conn.Close()
// Temporary fix for missing amqp entries
time.Sleep(10 * time.Second)
for i := 0; i < DefaultEntriesCount/5; i++ {
ch, err := conn.Channel()
if err != nil {
t.Errorf("failed to open a channel, err: %v", err)
return
}
exchangeName := "exchange"
err = ch.ExchangeDeclare(exchangeName, "direct", true, false, false, false, nil)
if err != nil {
t.Errorf("failed to declare an exchange, err: %v", err)
return
}
q, err := ch.QueueDeclare("queue", true, false, false, false, nil)
if err != nil {
t.Errorf("failed to declare a queue, err: %v", err)
return
}
routingKey := "routing_key"
err = ch.QueueBind(q.Name, routingKey, exchangeName, false, nil)
if err != nil {
t.Errorf("failed to bind the queue, err: %v", err)
return
}
err = ch.Publish(exchangeName, routingKey, false, false,
amqp.Publishing{
DeliveryMode: amqp.Persistent,
ContentType: "text/plain",
Body: []byte("message"),
})
if err != nil {
t.Errorf("failed to publish a message, err: %v", err)
return
}
msgChan, err := ch.Consume(q.Name, "Consumer", true, false, false, false, nil)
if err != nil {
t.Errorf("failed to create a consumer, err: %v", err)
return
}
select {
case <-msgChan:
break
case <-time.After(3 * time.Second):
t.Errorf("failed to consume a message on time")
return
}
err = ch.ExchangeDelete(exchangeName, false, false)
if err != nil {
t.Errorf("failed to delete the exchange, err: %v", err)
return
}
_, err = ch.QueueDelete(q.Name, false, false, false)
if err != nil {
t.Errorf("failed to delete the queue, err: %v", err)
return
}
ch.Close()
}
RunCypressTests(t, "npx cypress run --spec \"cypress/e2e/tests/Rabbit.js\"")
}

View File

@ -1,13 +1,56 @@
module github.com/up9inc/mizu/tests
module github.com/up9inc/mizu/acceptanceTests
go 1.16
go 1.17
require (
github.com/gorilla/websocket v1.4.2
github.com/go-redis/redis/v8 v8.11.4
github.com/rabbitmq/amqp091-go v1.3.0
github.com/up9inc/mizu/shared v0.0.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
k8s.io/apimachinery v0.23.3
k8s.io/client-go v0.23.3
)
require (
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-logr/logr v1.2.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/api v0.23.3 // indirect
k8s.io/klog/v2 v2.40.1 // indirect
k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf // indirect
k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect
sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)
replace github.com/up9inc/mizu/logger v0.0.0 => ../logger
replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api
replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../tap/dbgctl

File diff suppressed because it is too large Load Diff

View File

@ -11,22 +11,22 @@ func TestLogs(t *testing.T) {
t.Skip("ignored acceptance test")
}
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
tapCmdArgs := getDefaultTapCommandArgs()
tapCmdArgs := GetDefaultTapCommandArgs()
tapNamespace := getDefaultTapNamespace()
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
tapCmd := exec.Command(cliPath, tapCmdArgs...)
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := cleanupCommand(tapCmd); err != nil {
if err := CleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
})
@ -36,14 +36,14 @@ func TestLogs(t *testing.T) {
return
}
apiServerUrl := getApiServerUrl(defaultApiServerPort)
apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
if err := waitTapPodsReady(apiServerUrl); err != nil {
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
logsCmdArgs := getDefaultLogsCommandArgs()
logsCmdArgs := GetDefaultLogsCommandArgs()
logsCmd := exec.Command(cliPath, logsCmdArgs...)
t.Logf("running command: %v", logsCmd.String())
@ -58,7 +58,7 @@ func TestLogs(t *testing.T) {
return
}
logsPath, logsPathErr := getLogsPath()
logsPath, logsPathErr := GetLogsPath()
if logsPathErr != nil {
t.Errorf("failed to get logs path, err: %v", logsPathErr)
return
@ -112,22 +112,22 @@ func TestLogsPath(t *testing.T) {
t.Skip("ignored acceptance test")
}
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
tapCmdArgs := getDefaultTapCommandArgs()
tapCmdArgs := GetDefaultTapCommandArgs()
tapNamespace := getDefaultTapNamespace()
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
tapCmd := exec.Command(cliPath, tapCmdArgs...)
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := cleanupCommand(tapCmd); err != nil {
if err := CleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
})
@ -137,14 +137,14 @@ func TestLogsPath(t *testing.T) {
return
}
apiServerUrl := getApiServerUrl(defaultApiServerPort)
apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
if err := waitTapPodsReady(apiServerUrl); err != nil {
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
logsCmdArgs := getDefaultLogsCommandArgs()
logsCmdArgs := GetDefaultLogsCommandArgs()
logsPath := "../logs.zip"
logsCmdArgs = append(logsCmdArgs, "-f", logsPath)

61
acceptanceTests/setup.sh Normal file → Executable file
View File

@ -1,7 +1,10 @@
#!/bin/bash
set -e
PREFIX=$HOME/local/bin
VERSION=v1.22.0
TUNNEL_LOG="tunnel.log"
PROXY_LOG="proxy.log"
echo "Attempting to install minikube and assorted tools to $PREFIX"
@ -11,7 +14,7 @@ if ! [ -x "$(command -v kubectl)" ]; then
chmod +x kubectl
mv kubectl "$PREFIX"
else
echo "kubetcl is already installed"
echo "kubectl is already installed"
fi
if ! [ -x "$(command -v minikube)" ]; then
@ -24,32 +27,58 @@ else
fi
echo "Starting minikube..."
minikube start
minikube start --cpus 2 --memory 6000
echo "Creating mizu tests namespaces"
kubectl create namespace mizu-tests
kubectl create namespace mizu-tests2
kubectl create namespace mizu-tests --dry-run=client -o yaml | kubectl apply -f -
kubectl create namespace mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
echo "Creating httpbin deployments"
kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests
kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests
kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2
kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
echo "Creating redis deployment"
kubectl create deployment redis --image=redis -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
echo "Creating rabbitmq deployment"
kubectl create deployment rabbitmq --image=rabbitmq -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
echo "Creating httpbin services"
kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests
kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests
kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2
kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
echo "Creating redis service"
kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
echo "Creating rabbitmq service"
kubectl expose deployment rabbitmq --type=LoadBalancer --port=5672 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
echo "Starting proxy"
kubectl proxy --port=8080 &
rm -f ${PROXY_LOG}
kubectl proxy --port=8080 > ${PROXY_LOG} &
PID1=$!
echo "kubectl proxy process id is ${PID1} and log of proxy in ${PROXY_LOG}"
echo "Setting minikube docker env"
eval $(minikube docker-env)
if [[ -z "${CI}" ]]; then
echo "Setting env var of mizu ci image"
export MIZU_CI_IMAGE="mizu/ci:0.0"
echo "Build agent image"
docker build -t "${MIZU_CI_IMAGE}" .
else
echo "not building docker image in CI because it is created as separate step"
fi
echo "Build agent image"
make build-docker-ci
minikube image load "${MIZU_CI_IMAGE}"
echo "Build cli"
make build-cli-ci
cd cli && make build GIT_BRANCH=ci SUFFIX=ci
echo "Starting tunnel"
rm -f ${TUNNEL_LOG}
minikube tunnel > ${TUNNEL_LOG} &
PID2=$!
echo "Minikube tunnel process id is ${PID2} and log of tunnel in ${TUNNEL_LOG}"

View File

@ -2,11 +2,8 @@ package acceptanceTests
import (
"archive/zip"
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"os/exec"
"path"
"strings"
@ -15,6 +12,10 @@ import (
)
func TestTap(t *testing.T) {
basicTapTest(t, false)
}
func basicTapTest(t *testing.T, shouldCheckSrcAndDest bool, extraArgs... string) {
if testing.Short() {
t.Skip("ignored acceptance test")
}
@ -23,22 +24,24 @@ func TestTap(t *testing.T) {
for _, entriesCount := range tests {
t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
tapCmdArgs := getDefaultTapCommandArgs()
tapCmdArgs := GetDefaultTapCommandArgs()
tapNamespace := getDefaultTapNamespace()
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
tapCmdArgs = append(tapCmdArgs, extraArgs...)
tapCmd := exec.Command(cliPath, tapCmdArgs...)
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := cleanupCommand(tapCmd); err != nil {
if err := CleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
})
@ -48,50 +51,33 @@ func TestTap(t *testing.T) {
return
}
apiServerUrl := getApiServerUrl(defaultApiServerPort)
apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
if err := waitTapPodsReady(apiServerUrl); err != nil {
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
for i := 0; i < entriesCount; i++ {
if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
if _, requestErr := ExecuteHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
t.Errorf("failed to send proxy request, err: %v", requestErr)
return
}
}
entriesCheckFunc := func() error {
timestamp := time.Now().UnixNano() / int64(time.Millisecond)
entries, err := getDBEntries(timestamp, entriesCount, 1*time.Second)
if err != nil {
return err
}
err = checkEntriesAtLeast(entries, 1)
if err != nil {
return err
}
entry := entries[0]
entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entry["id"])
requestResult, requestErr := executeHttpGetRequest(entryUrl)
if requestErr != nil {
return fmt.Errorf("failed to get entry, err: %v", requestErr)
}
if requestResult == nil {
return fmt.Errorf("unexpected nil entry result")
}
return nil
expectedPods := []PodDescriptor{
{Name: "httpbin", Namespace: "mizu-tests"},
{Name: "httpbin2", Namespace: "mizu-tests"},
}
if err := retriesExecute(shortRetriesCount, entriesCheckFunc); err != nil {
t.Errorf("%v", err)
return
var expectedPodsStr string
for i := 0; i < len(expectedPods); i++ {
expectedPodsStr += fmt.Sprintf("Name:%vNamespace:%v", expectedPods[i].Name, expectedPods[i].Namespace)
}
RunCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/e2e/tests/UiTest.js\" --env entriesCount=%d,arrayDict=%v,shouldCheckSrcAndDest=%v",
entriesCount, expectedPodsStr, shouldCheckSrcAndDest))
})
}
}
@ -105,15 +91,15 @@ func TestTapGuiPort(t *testing.T) {
for _, guiPort := range tests {
t.Run(fmt.Sprintf("%d", guiPort), func(t *testing.T) {
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
tapCmdArgs := getDefaultTapCommandArgs()
tapCmdArgs := GetDefaultTapCommandArgs()
tapNamespace := getDefaultTapNamespace()
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
tapCmdArgs = append(tapCmdArgs, "-p", fmt.Sprintf("%d", guiPort))
@ -122,7 +108,7 @@ func TestTapGuiPort(t *testing.T) {
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := cleanupCommand(tapCmd); err != nil {
if err := CleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
})
@ -132,14 +118,23 @@ func TestTapGuiPort(t *testing.T) {
return
}
apiServerUrl := getApiServerUrl(guiPort)
apiServerUrl := GetApiServerUrl(guiPort)
if err := waitTapPodsReady(apiServerUrl); err != nil {
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/GuiPort.js\" --env port=%d", guiPort))
proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
for i := 0; i < DefaultEntriesCount; i++ {
if _, requestErr := ExecuteHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
t.Errorf("failed to send proxy request, err: %v", requestErr)
return
}
}
RunCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/e2e/tests/GuiPort.js\" --env name=%v,namespace=%v,port=%d",
"httpbin", "mizu-tests", guiPort))
})
}
}
@ -155,20 +150,20 @@ func TestTapAllNamespaces(t *testing.T) {
{Name: "httpbin", Namespace: "mizu-tests2"},
}
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
tapCmdArgs := getDefaultTapCommandArgs()
tapCmdArgs := GetDefaultTapCommandArgs()
tapCmdArgs = append(tapCmdArgs, "-A")
tapCmd := exec.Command(cliPath, tapCmdArgs...)
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := cleanupCommand(tapCmd); err != nil {
if err := CleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
})
@ -178,14 +173,14 @@ func TestTapAllNamespaces(t *testing.T) {
return
}
apiServerUrl := getApiServerUrl(defaultApiServerPort)
apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
if err := waitTapPodsReady(apiServerUrl); err != nil {
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
RunCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/e2e/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
}
@ -200,13 +195,13 @@ func TestTapMultipleNamespaces(t *testing.T) {
{Name: "httpbin", Namespace: "mizu-tests2"},
}
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
tapCmdArgs := getDefaultTapCommandArgs()
tapCmdArgs := GetDefaultTapCommandArgs()
var namespacesCmd []string
for _, expectedPod := range expectedPods {
namespacesCmd = append(namespacesCmd, "-n", expectedPod.Namespace)
@ -217,7 +212,7 @@ func TestTapMultipleNamespaces(t *testing.T) {
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := cleanupCommand(tapCmd); err != nil {
if err := CleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
})
@ -227,14 +222,14 @@ func TestTapMultipleNamespaces(t *testing.T) {
return
}
apiServerUrl := getApiServerUrl(defaultApiServerPort)
apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
if err := waitTapPodsReady(apiServerUrl); err != nil {
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
RunCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/e2e/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
}
@ -248,22 +243,22 @@ func TestTapRegex(t *testing.T) {
{Name: regexPodName, Namespace: "mizu-tests"},
}
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
tapCmdArgs := getDefaultTapCommandArgsWithRegex(regexPodName)
tapCmdArgs := GetDefaultTapCommandArgsWithRegex(regexPodName)
tapNamespace := getDefaultTapNamespace()
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
tapCmd := exec.Command(cliPath, tapCmdArgs...)
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := cleanupCommand(tapCmd); err != nil {
if err := CleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
})
@ -273,14 +268,14 @@ func TestTapRegex(t *testing.T) {
return
}
apiServerUrl := getApiServerUrl(defaultApiServerPort)
apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
if err := waitTapPodsReady(apiServerUrl); err != nil {
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/Regex.js\" --env name=%v,namespace=%v",
RunCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/e2e/tests/Regex.js\" --env name=%v,namespace=%v",
expectedPods[0].Name, expectedPods[0].Namespace))
}
@ -289,15 +284,15 @@ func TestTapDryRun(t *testing.T) {
t.Skip("ignored acceptance test")
}
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
tapCmdArgs := getDefaultTapCommandArgs()
tapCmdArgs := GetDefaultTapCommandArgs()
tapNamespace := getDefaultTapNamespace()
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
tapCmdArgs = append(tapCmdArgs, "--dry-run")
@ -321,7 +316,7 @@ func TestTapDryRun(t *testing.T) {
}()
go func() {
time.Sleep(shortRetriesCount * time.Second)
time.Sleep(ShortRetriesCount * time.Second)
resultChannel <- "fail"
}()
@ -336,22 +331,23 @@ func TestTapRedact(t *testing.T) {
t.Skip("ignored acceptance test")
}
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
tapCmdArgs := getDefaultTapCommandArgs()
tapCmdArgs := GetDefaultTapCommandArgs()
tapNamespace := getDefaultTapNamespace()
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
tapCmdArgs = append(tapCmdArgs, "--redact", "--set", "tap.redact-patterns.request-headers=User-Header", "--set", "tap.redact-patterns.request-body=User")
tapCmd := exec.Command(cliPath, tapCmdArgs...)
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := cleanupCommand(tapCmd); err != nil {
if err := CleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
})
@ -361,76 +357,24 @@ func TestTapRedact(t *testing.T) {
return
}
apiServerUrl := getApiServerUrl(defaultApiServerPort)
apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
if err := waitTapPodsReady(apiServerUrl); err != nil {
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
requestHeaders := map[string]string{"User-Header": "Mizu"}
requestBody := map[string]string{"User": "Mizu"}
for i := 0; i < defaultEntriesCount; i++ {
if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
for i := 0; i < DefaultEntriesCount; i++ {
if _, requestErr := ExecuteHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
t.Errorf("failed to send proxy request, err: %v", requestErr)
return
}
}
redactCheckFunc := func() error {
timestamp := time.Now().UnixNano() / int64(time.Millisecond)
entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
if err != nil {
return err
}
err = checkEntriesAtLeast(entries, 1)
if err != nil {
return err
}
firstEntry := entries[0]
entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
requestResult, requestErr := executeHttpGetRequest(entryUrl)
if requestErr != nil {
return fmt.Errorf("failed to get entry, err: %v", requestErr)
}
entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
request := entry["request"].(map[string]interface{})
headers := request["_headers"].([]interface{})
for _, headerInterface := range headers {
header := headerInterface.(map[string]interface{})
if header["name"].(string) != "User-Header" {
continue
}
userHeader := header["value"].(string)
if userHeader != "[REDACTED]" {
return fmt.Errorf("unexpected result - user agent is not redacted")
}
}
postData := request["postData"].(map[string]interface{})
textDataStr := postData["text"].(string)
var textData map[string]string
if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil {
return fmt.Errorf("failed to parse text data, err: %v", parseErr)
}
if textData["User"] != "[REDACTED]" {
return fmt.Errorf("unexpected result - user in body is not redacted")
}
return nil
}
if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
t.Errorf("%v", err)
return
}
RunCypressTests(t, "npx cypress run --spec \"cypress/e2e/tests/Redact.js\"")
}
func TestTapNoRedact(t *testing.T) {
@ -438,24 +382,22 @@ func TestTapNoRedact(t *testing.T) {
t.Skip("ignored acceptance test")
}
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
tapCmdArgs := getDefaultTapCommandArgs()
tapCmdArgs := GetDefaultTapCommandArgs()
tapNamespace := getDefaultTapNamespace()
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
tapCmdArgs = append(tapCmdArgs, "--no-redact")
tapCmd := exec.Command(cliPath, tapCmdArgs...)
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := cleanupCommand(tapCmd); err != nil {
if err := CleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
})
@ -465,161 +407,24 @@ func TestTapNoRedact(t *testing.T) {
return
}
apiServerUrl := getApiServerUrl(defaultApiServerPort)
apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
if err := waitTapPodsReady(apiServerUrl); err != nil {
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
requestHeaders := map[string]string{"User-Header": "Mizu"}
requestBody := map[string]string{"User": "Mizu"}
for i := 0; i < defaultEntriesCount; i++ {
if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
for i := 0; i < DefaultEntriesCount; i++ {
if _, requestErr := ExecuteHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
t.Errorf("failed to send proxy request, err: %v", requestErr)
return
}
}
redactCheckFunc := func() error {
timestamp := time.Now().UnixNano() / int64(time.Millisecond)
entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
if err != nil {
return err
}
err = checkEntriesAtLeast(entries, 1)
if err != nil {
return err
}
firstEntry := entries[0]
entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
requestResult, requestErr := executeHttpGetRequest(entryUrl)
if requestErr != nil {
return fmt.Errorf("failed to get entry, err: %v", requestErr)
}
entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
request := entry["request"].(map[string]interface{})
headers := request["_headers"].([]interface{})
for _, headerInterface := range headers {
header := headerInterface.(map[string]interface{})
if header["name"].(string) != "User-Header" {
continue
}
userHeader := header["value"].(string)
if userHeader == "[REDACTED]" {
return fmt.Errorf("unexpected result - user agent is redacted")
}
}
postData := request["postData"].(map[string]interface{})
textDataStr := postData["text"].(string)
var textData map[string]string
if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil {
return fmt.Errorf("failed to parse text data, err: %v", parseErr)
}
if textData["User"] == "[REDACTED]" {
return fmt.Errorf("unexpected result - user in body is redacted")
}
return nil
}
if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
t.Errorf("%v", err)
return
}
}
func TestTapRegexMasking(t *testing.T) {
if testing.Short() {
t.Skip("ignored acceptance test")
}
cliPath, cliPathErr := getCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
tapCmdArgs := getDefaultTapCommandArgs()
tapNamespace := getDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
tapCmdArgs = append(tapCmdArgs, "-r", "Mizu")
tapCmd := exec.Command(cliPath, tapCmdArgs...)
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := cleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
})
if err := tapCmd.Start(); err != nil {
t.Errorf("failed to start tap command, err: %v", err)
return
}
apiServerUrl := getApiServerUrl(defaultApiServerPort)
if err := waitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
for i := 0; i < defaultEntriesCount; i++ {
response, requestErr := http.Post(fmt.Sprintf("%v/post", proxyUrl), "text/plain", bytes.NewBufferString("Mizu"))
if _, requestErr = executeHttpRequest(response, requestErr); requestErr != nil {
t.Errorf("failed to send proxy request, err: %v", requestErr)
return
}
}
redactCheckFunc := func() error {
timestamp := time.Now().UnixNano() / int64(time.Millisecond)
entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
if err != nil {
return err
}
err = checkEntriesAtLeast(entries, 1)
if err != nil {
return err
}
firstEntry := entries[0]
entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
requestResult, requestErr := executeHttpGetRequest(entryUrl)
if requestErr != nil {
return fmt.Errorf("failed to get entry, err: %v", requestErr)
}
entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
request := entry["request"].(map[string]interface{})
postData := request["postData"].(map[string]interface{})
textData := postData["text"].(string)
if textData != "[REDACTED]" {
return fmt.Errorf("unexpected result - body is not redacted")
}
return nil
}
if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
t.Errorf("%v", err)
return
}
RunCypressTests(t, "npx cypress run --spec \"cypress/e2e/tests/NoRedact.js\"")
}
func TestTapIgnoredUserAgents(t *testing.T) {
@ -627,15 +432,15 @@ func TestTapIgnoredUserAgents(t *testing.T) {
t.Skip("ignored acceptance test")
}
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
tapCmdArgs := getDefaultTapCommandArgs()
tapCmdArgs := GetDefaultTapCommandArgs()
tapNamespace := getDefaultTapNamespace()
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
ignoredUserAgentValue := "ignore"
@ -645,7 +450,7 @@ func TestTapIgnoredUserAgents(t *testing.T) {
t.Logf("running command: %v", tapCmd.String())
t.Cleanup(func() {
if err := cleanupCommand(tapCmd); err != nil {
if err := CleanupCommand(tapCmd); err != nil {
t.Logf("failed to cleanup tap command, err: %v", err)
}
})
@ -655,70 +460,32 @@ func TestTapIgnoredUserAgents(t *testing.T) {
return
}
apiServerUrl := getApiServerUrl(defaultApiServerPort)
apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
if err := waitTapPodsReady(apiServerUrl); err != nil {
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
ignoredUserAgentCustomHeader := "Ignored-User-Agent"
headers := map[string]string{"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""}
for i := 0; i < defaultEntriesCount; i++ {
if _, requestErr := executeHttpGetRequestWithHeaders(fmt.Sprintf("%v/get", proxyUrl), headers); requestErr != nil {
for i := 0; i < DefaultEntriesCount; i++ {
if _, requestErr := ExecuteHttpGetRequestWithHeaders(fmt.Sprintf("%v/get", proxyUrl), headers); requestErr != nil {
t.Errorf("failed to send proxy request, err: %v", requestErr)
return
}
}
for i := 0; i < defaultEntriesCount; i++ {
if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
for i := 0; i < DefaultEntriesCount; i++ {
if _, requestErr := ExecuteHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
t.Errorf("failed to send proxy request, err: %v", requestErr)
return
}
}
ignoredUserAgentsCheckFunc := func() error {
timestamp := time.Now().UnixNano() / int64(time.Millisecond)
entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
if err != nil {
return err
}
err = checkEntriesAtLeast(entries, 1)
if err != nil {
return err
}
for _, entryInterface := range entries {
entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entryInterface["id"])
requestResult, requestErr := executeHttpGetRequest(entryUrl)
if requestErr != nil {
return fmt.Errorf("failed to get entry, err: %v", requestErr)
}
entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
request := entry["request"].(map[string]interface{})
headers := request["_headers"].([]interface{})
for _, headerInterface := range headers {
header := headerInterface.(map[string]interface{})
if header["name"].(string) != ignoredUserAgentCustomHeader {
continue
}
return fmt.Errorf("unexpected result - user agent is not ignored")
}
}
return nil
}
if err := retriesExecute(shortRetriesCount, ignoredUserAgentsCheckFunc); err != nil {
t.Errorf("%v", err)
return
}
RunCypressTests(t, "npx cypress run --spec \"cypress/e2e/tests/IgnoredUserAgents.js\"")
}
func TestTapDumpLogs(t *testing.T) {
@ -726,15 +493,15 @@ func TestTapDumpLogs(t *testing.T) {
t.Skip("ignored acceptance test")
}
cliPath, cliPathErr := getCliPath()
cliPath, cliPathErr := GetCliPath()
if cliPathErr != nil {
t.Errorf("failed to get cli path, err: %v", cliPathErr)
return
}
tapCmdArgs := getDefaultTapCommandArgs()
tapCmdArgs := GetDefaultTapCommandArgs()
tapNamespace := getDefaultTapNamespace()
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
tapCmdArgs = append(tapCmdArgs, "--set", "dump-logs=true")
@ -747,19 +514,19 @@ func TestTapDumpLogs(t *testing.T) {
return
}
apiServerUrl := getApiServerUrl(defaultApiServerPort)
apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
if err := waitTapPodsReady(apiServerUrl); err != nil {
if err := WaitTapPodsReady(apiServerUrl); err != nil {
t.Errorf("failed to start tap pods on time, err: %v", err)
return
}
if err := cleanupCommand(tapCmd); err != nil {
if err := CleanupCommand(tapCmd); err != nil {
t.Errorf("failed to cleanup tap command, err: %v", err)
return
}
mizuFolderPath, mizuPathErr := getMizuFolderPath()
mizuFolderPath, mizuPathErr := GetMizuFolderPath()
if mizuPathErr != nil {
t.Errorf("failed to get mizu folder path, err: %v", mizuPathErr)
return
@ -827,3 +594,44 @@ func TestTapDumpLogs(t *testing.T) {
return
}
}
func TestIpResolving(t *testing.T) {
namespace := AllNamespaces
t.Log("add permissions for ip-resolution for current user")
if err := ApplyKubeFilesForTest(
t,
"minikube",
namespace,
"../cli/cmd/permissionFiles/permissions-all-namespaces-ip-resolution-optional.yaml",
); err != nil {
t.Errorf("failed to create k8s permissions, %v", err)
return
}
basicTapTest(t, true)
}
func TestRestrictedMode(t *testing.T) {
namespace := "mizu-tests"
t.Log("creating permissions for restricted user")
if err := ApplyKubeFilesForTest(
t,
"minikube",
namespace,
"../cli/cmd/permissionFiles/permissions-ns-tap.yaml",
); err != nil {
t.Errorf("failed to create k8s permissions, %v", err)
return
}
t.Log("switching k8s context to user")
if err := SwitchKubeContextForTest(t, "user-with-restricted-access"); err != nil {
t.Errorf("failed to switch k8s context, %v", err)
return
}
extraArgs := []string{"--set", fmt.Sprintf("mizu-resources-namespace=%s", namespace)}
t.Run("basic tap", func (testingT *testing.T) {basicTapTest(testingT, false, extraArgs...)})
}

View File

@ -2,33 +2,37 @@ package acceptanceTests
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"sync"
"syscall"
"testing"
"time"
"github.com/gorilla/websocket"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
"github.com/up9inc/mizu/shared"
)
const (
longRetriesCount = 100
shortRetriesCount = 10
defaultApiServerPort = shared.DefaultApiServerPort
defaultNamespaceName = "mizu-tests"
defaultServiceName = "httpbin"
defaultEntriesCount = 50
waitAfterTapPodsReady = 3 * time.Second
cleanCommandTimeout = 1 * time.Minute
LongRetriesCount = 100
ShortRetriesCount = 10
DefaultApiServerPort = shared.DefaultApiServerPort
DefaultNamespaceName = "mizu-tests"
DefaultServiceName = "httpbin"
DefaultEntriesCount = 50
WaitAfterTapPodsReady = 3 * time.Second
AllNamespaces = ""
)
type PodDescriptor struct {
@ -36,19 +40,7 @@ type PodDescriptor struct {
Namespace string
}
func isPodDescriptorInPodArray(pods []map[string]interface{}, podDescriptor PodDescriptor) bool {
for _, pod := range pods {
podNamespace := pod["namespace"].(string)
podName := pod["name"].(string)
if podDescriptor.Namespace == podNamespace && strings.Contains(podName, podDescriptor.Name) {
return true
}
}
return false
}
func getCliPath() (string, error) {
func GetCliPath() (string, error) {
dir, filePathErr := os.Getwd()
if filePathErr != nil {
return "", filePathErr
@ -58,7 +50,7 @@ func getCliPath() (string, error) {
return cliPath, nil
}
func getMizuFolderPath() (string, error) {
func GetMizuFolderPath() (string, error) {
home, homeDirErr := os.UserHomeDir()
if homeDirErr != nil {
return "", homeDirErr
@ -67,8 +59,8 @@ func getMizuFolderPath() (string, error) {
return path.Join(home, ".mizu"), nil
}
func getConfigPath() (string, error) {
mizuFolderPath, mizuPathError := getMizuFolderPath()
func GetConfigPath() (string, error) {
mizuFolderPath, mizuPathError := GetMizuFolderPath()
if mizuPathError != nil {
return "", mizuPathError
}
@ -76,90 +68,209 @@ func getConfigPath() (string, error) {
return path.Join(mizuFolderPath, "config.yaml"), nil
}
func getProxyUrl(namespace string, service string) string {
func GetProxyUrl(namespace string, service string) string {
return fmt.Sprintf("http://localhost:8080/api/v1/namespaces/%v/services/%v/proxy", namespace, service)
}
func getApiServerUrl(port uint16) string {
func GetApiServerUrl(port uint16) string {
return fmt.Sprintf("http://localhost:%v", port)
}
func getWebSocketUrl(port uint16) string {
return fmt.Sprintf("ws://localhost:%v/ws", port)
func NewKubernetesProvider() (*KubernetesProvider, error) {
home := homedir.HomeDir()
configLoadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(home, ".kube", "config")}
clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
configLoadingRules,
&clientcmd.ConfigOverrides{
CurrentContext: "",
},
)
restClientConfig, err := clientConfig.ClientConfig()
if err != nil {
return nil, err
}
clientSet, err := kubernetes.NewForConfig(restClientConfig)
if err != nil {
return nil, err
}
return &KubernetesProvider{clientSet}, nil
}
type KubernetesProvider struct {
clientSet *kubernetes.Clientset
}
func (kp *KubernetesProvider) GetServiceExternalIp(ctx context.Context, namespace string, service string) (string, error) {
serviceObj, err := kp.clientSet.CoreV1().Services(namespace).Get(ctx, service, metav1.GetOptions{})
if err != nil {
return "", err
}
externalIp := serviceObj.Status.LoadBalancer.Ingress[0].IP
return externalIp, nil
}
func SwitchKubeContextForTest(t *testing.T, newContextName string) error {
prevKubeContextName, err := GetKubeCurrentContextName()
if err != nil {
return err
}
if err := SetKubeCurrentContext(newContextName); err != nil {
return err
}
t.Cleanup(func() {
if err := SetKubeCurrentContext(prevKubeContextName); err != nil {
t.Errorf("failed to set Kubernetes context to %s, err: %v", prevKubeContextName, err)
t.Errorf("cleanup failed, subsequent tests may be affected")
}
})
return nil
}
func GetKubeCurrentContextName() (string, error) {
cmd := exec.Command("kubectl", "config", "current-context")
output, err := cmd.CombinedOutput()
if err != nil {
return "", fmt.Errorf("%v, %s", err, string(output))
}
return string(bytes.TrimSpace(output)), nil
}
func SetKubeCurrentContext(contextName string) error {
cmd := exec.Command("kubectl", "config", "use-context", contextName)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("%v, %s", err, string(output))
}
return nil
}
func ApplyKubeFilesForTest(t *testing.T, kubeContext string, namespace string, filename ...string) error {
for i := range filename {
fname := filename[i]
if err := ApplyKubeFile(kubeContext, namespace, fname); err != nil {
return err
}
t.Cleanup(func() {
if err := DeleteKubeFile(kubeContext, namespace, fname); err != nil {
t.Errorf(
"failed to delete Kubernetes resources in namespace %s from filename %s, err: %v",
namespace,
fname,
err,
)
}
})
}
return nil
}
func ApplyKubeFile(kubeContext string, namespace string, filename string) error {
cmdArgs := []string{
"apply",
"--context", kubeContext,
"-f", filename,
}
if namespace != AllNamespaces {
cmdArgs = append(cmdArgs, "-n", namespace)
}
cmd := exec.Command("kubectl", cmdArgs...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("%v, %s", err, string(output))
}
return nil
}
func DeleteKubeFile(kubeContext string, namespace string, filename string) error {
cmdArgs := []string{
"delete",
"--context", kubeContext,
"-f", filename,
}
if namespace != AllNamespaces {
cmdArgs = append(cmdArgs, "-n", namespace)
}
cmd := exec.Command("kubectl", cmdArgs...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("%v, %s", err, string(output))
}
return nil
}
func getDefaultCommandArgs() []string {
agentImageValue := os.Getenv("MIZU_CI_IMAGE")
setFlag := "--set"
telemetry := "telemetry=false"
agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0"
agentImage := fmt.Sprintf("agent-image=%s", agentImageValue)
imagePullPolicy := "image-pull-policy=IfNotPresent"
headless := "headless=true"
return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
return []string{setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
}
func getDefaultTapCommandArgs() []string {
func GetDefaultTapCommandArgs() []string {
tapCommand := "tap"
defaultCmdArgs := getDefaultCommandArgs()
return append([]string{tapCommand}, defaultCmdArgs...)
}
func getDefaultTapCommandArgsWithRegex(regex string) []string {
func GetDefaultTapCommandArgsWithRegex(regex string) []string {
tapCommand := "tap"
defaultCmdArgs := getDefaultCommandArgs()
return append([]string{tapCommand, regex}, defaultCmdArgs...)
}
func getDefaultLogsCommandArgs() []string {
func GetDefaultLogsCommandArgs() []string {
logsCommand := "logs"
defaultCmdArgs := getDefaultCommandArgs()
return append([]string{logsCommand}, defaultCmdArgs...)
}
func getDefaultTapNamespace() []string {
func GetDefaultTapNamespace() []string {
return []string{"-n", "mizu-tests"}
}
func getDefaultConfigCommandArgs() []string {
func GetDefaultConfigCommandArgs() []string {
configCommand := "config"
defaultCmdArgs := getDefaultCommandArgs()
return append([]string{configCommand}, defaultCmdArgs...)
}
func getDefaultCleanCommandArgs() []string {
cleanCommand := "clean"
defaultCmdArgs := getDefaultCommandArgs()
return append([]string{cleanCommand}, defaultCmdArgs...)
}
func getDefaultViewCommandArgs() []string {
viewCommand := "view"
defaultCmdArgs := getDefaultCommandArgs()
return append([]string{viewCommand}, defaultCmdArgs...)
}
func runCypressTests(t *testing.T, cypressRunCmd string) {
func RunCypressTests(t *testing.T, cypressRunCmd string) {
cypressCmd := exec.Command("bash", "-c", cypressRunCmd)
t.Logf("running command: %v", cypressCmd.String())
out, err := cypressCmd.Output()
out, err := cypressCmd.CombinedOutput()
if err != nil {
t.Errorf("%s", out)
t.Errorf("error running cypress, error: %v, output: %v", err, string(out))
return
}
t.Logf("%s", out)
}
func retriesExecute(retriesCount int, executeFunc func() error) error {
func RetriesExecute(retriesCount int, executeFunc func() error) error {
var lastError interface{}
for i := 0; i < retriesCount; i++ {
if err := tryExecuteFunc(executeFunc); err != nil {
if err := TryExecuteFunc(executeFunc); err != nil {
lastError = err
time.Sleep(1 * time.Second)
@ -172,7 +283,7 @@ func retriesExecute(retriesCount int, executeFunc func() error) error {
return fmt.Errorf("reached max retries count, retries count: %v, last err: %v", retriesCount, lastError)
}
func tryExecuteFunc(executeFunc func() error) (err interface{}) {
func TryExecuteFunc(executeFunc func() error) (err interface{}) {
defer func() {
if panicErr := recover(); panicErr != nil {
err = panicErr
@ -182,26 +293,26 @@ func tryExecuteFunc(executeFunc func() error) (err interface{}) {
return executeFunc()
}
func waitTapPodsReady(apiServerUrl string) error {
resolvingUrl := fmt.Sprintf("%v/status/tappersCount", apiServerUrl)
func WaitTapPodsReady(apiServerUrl string) error {
resolvingUrl := fmt.Sprintf("%v/status/connectedTappersCount", apiServerUrl)
tapPodsReadyFunc := func() error {
requestResult, requestErr := executeHttpGetRequest(resolvingUrl)
requestResult, requestErr := ExecuteHttpGetRequest(resolvingUrl)
if requestErr != nil {
return requestErr
}
tappersCount := requestResult.(float64)
if tappersCount == 0 {
return fmt.Errorf("no tappers running")
connectedTappersCount := requestResult.(float64)
if connectedTappersCount == 0 {
return fmt.Errorf("no connected tappers running")
}
time.Sleep(waitAfterTapPodsReady)
time.Sleep(WaitAfterTapPodsReady)
return nil
}
return retriesExecute(longRetriesCount, tapPodsReadyFunc)
return RetriesExecute(LongRetriesCount, tapPodsReadyFunc)
}
func jsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
func JsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
var result interface{}
if parseErr := json.Unmarshal(jsonBytes, &result); parseErr != nil {
return nil, parseErr
@ -210,7 +321,7 @@ func jsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
return result, nil
}
func executeHttpRequest(response *http.Response, requestErr error) (interface{}, error) {
func ExecuteHttpRequest(response *http.Response, requestErr error) (interface{}, error) {
if requestErr != nil {
return nil, requestErr
} else if response.StatusCode != 200 {
@ -224,10 +335,10 @@ func executeHttpRequest(response *http.Response, requestErr error) (interface{},
return nil, readErr
}
return jsonBytesToInterface(data)
return JsonBytesToInterface(data)
}
func executeHttpGetRequestWithHeaders(url string, headers map[string]string) (interface{}, error) {
func ExecuteHttpGetRequestWithHeaders(url string, headers map[string]string) (interface{}, error) {
request, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
@ -239,15 +350,15 @@ func executeHttpGetRequestWithHeaders(url string, headers map[string]string) (in
client := &http.Client{}
response, requestErr := client.Do(request)
return executeHttpRequest(response, requestErr)
return ExecuteHttpRequest(response, requestErr)
}
func executeHttpGetRequest(url string) (interface{}, error) {
func ExecuteHttpGetRequest(url string) (interface{}, error) {
response, requestErr := http.Get(url)
return executeHttpRequest(response, requestErr)
return ExecuteHttpRequest(response, requestErr)
}
func executeHttpPostRequestWithHeaders(url string, headers map[string]string, body interface{}) (interface{}, error) {
func ExecuteHttpPostRequestWithHeaders(url string, headers map[string]string, body interface{}) (interface{}, error) {
requestBody, jsonErr := json.Marshal(body)
if jsonErr != nil {
return nil, jsonErr
@ -265,40 +376,10 @@ func executeHttpPostRequestWithHeaders(url string, headers map[string]string, bo
client := &http.Client{}
response, requestErr := client.Do(request)
return executeHttpRequest(response, requestErr)
return ExecuteHttpRequest(response, requestErr)
}
func runMizuClean() error {
cliPath, err := getCliPath()
if err != nil {
return err
}
cleanCmdArgs := getDefaultCleanCommandArgs()
cleanCmd := exec.Command(cliPath, cleanCmdArgs...)
commandDone := make(chan error)
go func() {
if err := cleanCmd.Run(); err != nil {
commandDone <- err
}
commandDone <- nil
}()
select {
case err = <-commandDone:
if err != nil {
return err
}
case <-time.After(cleanCommandTimeout):
return errors.New("clean command timed out")
}
return nil
}
func cleanupCommand(cmd *exec.Cmd) error {
func CleanupCommand(cmd *exec.Cmd) error {
if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil {
return err
}
@ -310,18 +391,7 @@ func cleanupCommand(cmd *exec.Cmd) error {
return nil
}
func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) {
tapPodsInterface := tapStatusInterface.([]interface{})
var pods []map[string]interface{}
for _, podInterface := range tapPodsInterface {
pods = append(pods, podInterface.(map[string]interface{}))
}
return pods, nil
}
func getLogsPath() (string, error) {
func GetLogsPath() (string, error) {
dir, filePathErr := os.Getwd()
if filePathErr != nil {
return "", filePathErr
@ -331,77 +401,6 @@ func getLogsPath() (string, error) {
return logsPath, nil
}
// waitTimeout waits for the waitgroup for the specified max timeout.
// Returns true if waiting timed out.
func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
channel := make(chan struct{})
go func() {
defer close(channel)
wg.Wait()
}()
select {
case <-channel:
return false // completed normally
case <-time.After(timeout):
return true // timed out
}
}
// checkEntriesAtLeast checks whether the number of entries greater than or equal to n
func checkEntriesAtLeast(entries []map[string]interface{}, n int) error {
if len(entries) < n {
return fmt.Errorf("Unexpected entries result - Expected more than %d entries", n-1)
}
return nil
}
// getDBEntries retrieves the entries from the database before the given timestamp.
// Also limits the results according to the limit parameter.
// Timeout for the WebSocket connection is defined by the timeout parameter.
func getDBEntries(timestamp int64, limit int, timeout time.Duration) (entries []map[string]interface{}, err error) {
query := fmt.Sprintf("timestamp < %d and limit(%d)", timestamp, limit)
webSocketUrl := getWebSocketUrl(defaultApiServerPort)
var connection *websocket.Conn
connection, _, err = websocket.DefaultDialer.Dial(webSocketUrl, nil)
if err != nil {
return
}
defer connection.Close()
handleWSConnection := func(wg *sync.WaitGroup) {
defer wg.Done()
for {
_, message, err := connection.ReadMessage()
if err != nil {
return
}
var data map[string]interface{}
if err = json.Unmarshal([]byte(message), &data); err != nil {
return
}
if data["messageType"] == "entry" {
entries = append(entries, data)
}
}
}
err = connection.WriteMessage(websocket.TextMessage, []byte(query))
if err != nil {
return
}
var wg sync.WaitGroup
go handleWSConnection(&wg)
wg.Add(1)
waitTimeout(&wg, timeout)
return
}
func Contains(slice []string, containsValue string) bool {
for _, sliceValue := range slice {
if sliceValue == containsValue {

View File

@ -1,20 +0,0 @@
# mizu agent
Agent for MIZU (API server and tapper)
Basic APIs:
* /stats - retrieve statistics of collected data
* /viewer - web ui
## Remote Debugging
### Setup remote debugging
1. Run `go get github.com/go-delve/delve/cmd/dlv`
2. Create a "Go Remote" run/debug configuration in Intellij, set to localhost:2345
3. Build and push a debug image using
`docker build . -t gcr.io/up9-docker-hub/mizu/debug:latest -f debug.Dockerfile && docker push gcr.io/up9-docker-hub/mizu/debug:latest`
### Connecting
1. Start mizu using the cli with the debug
image `mizu tap --set agent-image=gcr.io/up9-docker-hub/mizu/debug:latest {tapped_pod_name}`
2. Forward the debug port using `kubectl port-forward -n default mizu-api-server 2345:2345`
3. Run the run/debug configuration you've created earlier in Intellij.
<small>Do note that dlv won't start the api until a debugger connects to it.</small>

View File

@ -1,35 +1,163 @@
module mizuserver
module github.com/up9inc/mizu/agent
go 1.16
go 1.17
require (
github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
github.com/djherbis/atime v1.0.0
github.com/getkin/kin-openapi v0.76.0
github.com/chanced/openapi v0.0.8
github.com/djherbis/atime v1.1.0
github.com/gin-contrib/pprof v1.3.0
github.com/gin-contrib/static v0.0.1
github.com/gin-gonic/gin v1.7.7
github.com/go-playground/locales v0.13.0
github.com/go-playground/universal-translator v0.17.0
github.com/go-playground/validator/v10 v10.5.0
github.com/google/martian v2.1.0+incompatible
github.com/go-playground/locales v0.14.0
github.com/go-playground/universal-translator v0.18.0
github.com/go-playground/validator/v10 v10.10.0
github.com/google/uuid v1.3.0
github.com/gorilla/websocket v1.4.2
github.com/jinzhu/copier v0.3.5
github.com/nav-inc/datetime v0.1.3
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
github.com/ory/kratos-client-go v0.8.2-alpha.1
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d
github.com/orcaman/concurrent-map v1.0.0
github.com/stretchr/testify v1.7.0
github.com/up9inc/basenine/client/go v0.0.0-20220612112747-3b28eeac9c51
github.com/up9inc/mizu/logger v0.0.0
github.com/up9inc/mizu/shared v0.0.0
github.com/up9inc/mizu/tap v0.0.0
github.com/up9inc/mizu/tap/api v0.0.0
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
golang.org/x/text v0.3.5 // indirect
k8s.io/api v0.21.2
k8s.io/apimachinery v0.21.2
k8s.io/client-go v0.21.2
github.com/up9inc/mizu/tap/dbgctl v0.0.0
github.com/up9inc/mizu/tap/extensions/amqp v0.0.0
github.com/up9inc/mizu/tap/extensions/http v0.0.0
github.com/up9inc/mizu/tap/extensions/kafka v0.0.0
github.com/up9inc/mizu/tap/extensions/redis v0.0.0
github.com/wI2L/jsondiff v0.1.1
k8s.io/api v0.23.3
k8s.io/apimachinery v0.23.3
k8s.io/client-go v0.23.3
)
require (
cloud.google.com/go/compute v1.2.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest v0.11.24 // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/MakeNowJust/heredoc v1.0.0 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b // indirect
github.com/cilium/ebpf v0.9.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
github.com/fatih/camelcase v1.0.0 // indirect
github.com/fvbommel/sortorder v1.0.2 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-logr/logr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.6 // indirect
github.com/go-openapi/swag v0.21.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/martian v2.1.0+incompatible // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.14.2 // indirect
github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mertyildiran/gqlparser/v2 v2.4.6 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/moby/moby v20.10.17+incompatible // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/ohler55/ojg v1.12.12 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/russross/blackfriday v1.6.0 // indirect
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
github.com/segmentio/kafka-go v0.4.27 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/spf13/cobra v1.3.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/struCoder/pidusage v0.2.1 // indirect
github.com/tidwall/gjson v1.14.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tidwall/sjson v1.2.4 // indirect
github.com/tklauser/go-sysconf v0.3.10 // indirect
github.com/tklauser/numcpus v0.4.0 // indirect
github.com/ugorji/go/codec v1.2.6 // indirect
github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
github.com/xlab/treeprint v1.1.0 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.starlark.net v0.0.0-20220203230714-bb14e151c28f // indirect
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b // indirect
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
k8s.io/cli-runtime v0.23.3 // indirect
k8s.io/component-base v0.23.3 // indirect
k8s.io/klog/v2 v2.40.1 // indirect
k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf // indirect
k8s.io/kubectl v0.23.3 // indirect
k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect
sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
sigs.k8s.io/kustomize/api v0.11.1 // indirect
sigs.k8s.io/kustomize/kyaml v0.13.3 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)
replace github.com/up9inc/mizu/logger v0.0.0 => ../logger
replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
replace github.com/up9inc/mizu/tap v0.0.0 => ../tap
replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api
replace github.com/up9inc/mizu/tap/extensions/amqp v0.0.0 => ../tap/extensions/amqp
replace github.com/up9inc/mizu/tap/extensions/http v0.0.0 => ../tap/extensions/http
replace github.com/up9inc/mizu/tap/extensions/kafka v0.0.0 => ../tap/extensions/kafka
replace github.com/up9inc/mizu/tap/extensions/redis v0.0.0 => ../tap/extensions/redis
replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../tap/dbgctl

File diff suppressed because it is too large Load Diff

View File

@ -1,14 +0,0 @@
FROM gcr.io/up9-docker-hub/mizu-kratos-base/simple-password-policy:latest
USER root
RUN apk add sqlite
RUN mkdir -p /etc/config/kratos
COPY ./kratos.yml /etc/config/kratos/kratos.yml
COPY ./identity.schema.json /etc/config/kratos/identity.schema.json
COPY ./start.sh /opt/start.sh
RUN chmod +x /opt/start.sh
ENTRYPOINT ["/opt/start.sh"]

View File

@ -1,28 +0,0 @@
#!/bin/bash
set -e
GCP_PROJECT=up9-docker-hub
REPOSITORY=gcr.io/$GCP_PROJECT
SERVER_NAME=mizu-kratos
GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
SEM_VER=${SEM_VER=0.0.0}
DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
then
echo "Pushing to $GIT_BRANCH is allowed only via CI"
exit 1
fi
echo "building ${DOCKER_TAGGED_BUILDS[@]}"
DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
do
echo pushing "$DOCKER_TAG"
docker push "$DOCKER_TAG"
done

View File

@ -1,43 +0,0 @@
{
"$id": "https://schemas.ory.sh/presets/kratos/quickstart/email-password/identity.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Person",
"type": "object",
"properties": {
"traits": {
"type": "object",
"properties": {
"username": {
"type": "string",
"format": "username",
"title": "Username",
"minLength": 3,
"ory.sh/kratos": {
"credentials": {
"password": {
"identifier": true
}
}
}
},
"name": {
"type": "object",
"properties": {
"first": {
"title": "First Name",
"type": "string"
},
"last": {
"title": "Last Name",
"type": "string"
}
}
}
},
"required": [
"username"
],
"additionalProperties": false
}
}
}

View File

@ -1,84 +0,0 @@
version: v0.8.2-alpha.1
dsn: sqlite:///app/data/kratos.sqlite?_fk=true
serve:
public:
base_url: http://127.0.0.1:4433/
cors:
enabled: true
admin:
base_url: http://kratos:4434/
selfservice:
default_browser_return_url: http://127.0.0.1:4455/
whitelisted_return_urls:
- http://127.0.0.1:4455
methods:
password:
enabled: true
flows:
error:
ui_url: http://127.0.0.1:4455/error
settings:
ui_url: http://127.0.0.1:4455/settings
privileged_session_max_age: 15m
recovery:
enabled: true
ui_url: http://127.0.0.1:4455/recovery
verification:
enabled: false
ui_url: http://127.0.0.1:4455/verification
after:
default_browser_return_url: http://127.0.0.1:4455/
logout:
after:
default_browser_return_url: http://127.0.0.1:4455/login
login:
ui_url: http://127.0.0.1:4455/login
lifespan: 10m
registration:
lifespan: 10m
ui_url: http://127.0.0.1:4455/registration
after:
password:
hooks:
-
hook: session
log:
level: info
format: text
leak_sensitive_values: true
secrets:
cookie:
- PLEASE-CHANGE-ME-I-AM-VERY-INSECURE
cipher:
- 32-LONG-SECRET-NOT-SECURE-AT-ALL
ciphers:
algorithm: xchacha20-poly1305
hashers:
argon2:
parallelism: 1
memory: 128MB
iterations: 2
salt_length: 16
key_length: 16
identity:
default_schema_url: file:///etc/config/kratos/identity.schema.json
courier:
smtp:
connection_uri: smtps://test:test@mailslurper:1025/?skip_ssl_verify=true

View File

@ -1,4 +0,0 @@
#!/bin/sh
kratos migrate sql sqlite:///app/data/kratos.sqlite?_fk=true --yes # this initializes the db
kratos serve -c /etc/config/kratos/kratos.yml --watch-courier # start kratos

View File

@ -6,38 +6,37 @@ import (
"flag"
"fmt"
"io/ioutil"
"mizuserver/pkg/api"
"mizuserver/pkg/config"
"mizuserver/pkg/controllers"
"mizuserver/pkg/middlewares"
"mizuserver/pkg/models"
"mizuserver/pkg/routes"
"mizuserver/pkg/up9"
"mizuserver/pkg/utils"
"net/http"
"os"
"os/signal"
"path"
"path/filepath"
"plugin"
"sort"
"strconv"
"strings"
"syscall"
"time"
v1 "k8s.io/api/core/v1"
"github.com/antelman107/net-wait-go/wait"
"github.com/gin-contrib/pprof"
"github.com/gin-contrib/static"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/agent/pkg/dependency"
"github.com/up9inc/mizu/agent/pkg/entries"
"github.com/up9inc/mizu/agent/pkg/middlewares"
"github.com/up9inc/mizu/agent/pkg/models"
"github.com/up9inc/mizu/agent/pkg/oas"
"github.com/up9inc/mizu/agent/pkg/routes"
"github.com/up9inc/mizu/agent/pkg/servicemap"
"github.com/up9inc/mizu/agent/pkg/utils"
"github.com/up9inc/mizu/agent/pkg/api"
"github.com/up9inc/mizu/agent/pkg/app"
"github.com/up9inc/mizu/agent/pkg/config"
"github.com/gorilla/websocket"
"github.com/op/go-logging"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/logger"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/tap"
tapApi "github.com/up9inc/mizu/tap/api"
"github.com/up9inc/mizu/tap/dbgctl"
)
var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API")
@ -47,98 +46,41 @@ var apiServerAddress = flag.String("api-server-address", "", "Address of mizu AP
var namespace = flag.String("namespace", "", "Resolve IPs if they belong to resources in this namespace (default is all)")
var harsReaderMode = flag.Bool("hars-read", false, "Run in hars-read mode")
var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
var extensions []*tapApi.Extension // global
var extensionsMap map[string]*tapApi.Extension // global
var startTime int64
var profiler = flag.Bool("profiler", false, "Run pprof server")
const (
socketConnectionRetries = 30
socketConnectionRetryDelay = time.Second * 2
socketHandshakeTimeout = time.Second * 2
uiIndexPath = "./site/index.html"
)
func main() {
initializeDependencies()
logLevel := determineLogLevel()
logger.InitLoggerStderrOnly(logLevel)
logger.InitLoggerStd(logLevel)
flag.Parse()
if err := config.LoadConfig(); err != nil {
logger.Log.Fatalf("Error loading config file %v", err)
}
loadExtensions()
app.LoadExtensions()
if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode {
panic("One of the flags --tap, --api or --standalone or --hars-read must be provided")
panic("One of the flags --tap, --api-server, --standalone or --hars-read must be provided")
}
if *standaloneMode {
api.StartResolving(*namespace)
outputItemsChannel := make(chan *tapApi.OutputChannelItem)
filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
filteringOptions := getTrafficFilteringOptions()
hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
tapOpts := &tap.TapOpts{HostMode: hostMode}
tap.StartPassiveTapper(tapOpts, outputItemsChannel, extensions, filteringOptions)
go filterItems(outputItemsChannel, filteredOutputItemsChannel)
go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
hostApi(nil)
runInStandaloneMode()
} else if *tapperMode {
logger.Log.Infof("Starting tapper, websocket address: %s", *apiServerAddress)
if *apiServerAddress == "" {
panic("API server address must be provided with --api-server-address when using --tap")
}
hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
tapOpts := &tap.TapOpts{HostMode: hostMode}
tapTargets := getTapTargets()
if tapTargets != nil {
tapOpts.FilterAuthorities = tapTargets
logger.Log.Infof("Filtering for the following authorities: %v", tapOpts.FilterAuthorities)
}
filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
filteringOptions := getTrafficFilteringOptions()
tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, extensions, filteringOptions)
socketConnection, err := dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay)
if err != nil {
panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
}
logger.Log.Infof("Connected successfully to websocket %s", *apiServerAddress)
go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
runInTapperMode()
} else if *apiServerMode {
configureBasenineServer(shared.BasenineHost, shared.BaseninePort)
startTime = time.Now().UnixNano() / int64(time.Millisecond)
api.StartResolving(*namespace)
ginApp := runInApiServerMode(*namespace)
outputItemsChannel := make(chan *tapApi.OutputChannelItem)
filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
go filterItems(outputItemsChannel, filteredOutputItemsChannel)
go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
syncEntriesConfig := getSyncEntriesConfig()
if syncEntriesConfig != nil {
if err := up9.SyncEntries(syncEntriesConfig); err != nil {
panic(fmt.Sprintf("Error syncing entries, err: %v", err))
}
if *profiler {
pprof.Register(ginApp)
}
hostApi(outputItemsChannel)
} else if *harsReaderMode {
outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000)
filteredHarChannel := make(chan *tapApi.OutputChannelItem)
utils.StartServer(ginApp)
go filterItems(outputItemsChannel, filteredHarChannel)
go api.StartReadingEntries(filteredHarChannel, harsDir, extensionsMap)
hostApi(nil)
} else if *harsReaderMode {
runInHarReaderMode()
}
signalChan := make(chan os.Signal, 1)
@ -148,116 +90,128 @@ func main() {
logger.Log.Info("Exiting")
}
func configureBasenineServer(host string, port string) {
if !wait.New(
wait.WithProto("tcp"),
wait.WithWait(200*time.Millisecond),
wait.WithBreak(50*time.Millisecond),
wait.WithDeadline(5*time.Second),
wait.WithDebug(true),
).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
logger.Log.Panicf("Basenine is not available!")
}
func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) *gin.Engine {
ginApp := gin.Default()
// Limit the database size to default 200MB
err := basenine.Limit(host, port, config.Config.MaxDBSizeBytes)
if err != nil {
logger.Log.Panicf("Error while limiting database size: %v", err)
}
// Define the macros
for _, extension := range extensions {
macros := extension.Dissector.Macros()
for macro, expanded := range macros {
err = basenine.Macro(host, port, macro, expanded)
if err != nil {
logger.Log.Panicf("Error while adding a macro: %v", err)
}
}
}
}
func loadExtensions() {
dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
extensionsDir := path.Join(dir, "./extensions/")
files, err := ioutil.ReadDir(extensionsDir)
if err != nil {
logger.Log.Fatal(err)
}
extensions = make([]*tapApi.Extension, len(files))
extensionsMap = make(map[string]*tapApi.Extension)
for i, file := range files {
filename := file.Name()
logger.Log.Infof("Loading extension: %s", filename)
extension := &tapApi.Extension{
Path: path.Join(extensionsDir, filename),
}
plug, _ := plugin.Open(extension.Path)
extension.Plug = plug
symDissector, err := plug.Lookup("Dissector")
var dissector tapApi.Dissector
var ok bool
dissector, ok = symDissector.(tapApi.Dissector)
if err != nil || !ok {
panic(fmt.Sprintf("Failed to load the extension: %s", extension.Path))
}
dissector.Register(extension)
extension.Dissector = dissector
extensions[i] = extension
extensionsMap[extension.Protocol.Name] = extension
}
sort.Slice(extensions, func(i, j int) bool {
return extensions[i].Protocol.Priority < extensions[j].Protocol.Priority
})
for _, extension := range extensions {
logger.Log.Infof("Extension Properties: %+v", extension)
}
controllers.InitExtensionsMap(extensionsMap)
}
func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
app := gin.Default()
app.GET("/echo", func(c *gin.Context) {
c.String(http.StatusOK, "Here is Mizu agent")
ginApp.GET("/echo", func(c *gin.Context) {
c.JSON(http.StatusOK, "Here is Mizu agent")
})
eventHandlers := api.RoutesEventHandlers{
SocketOutChannel: socketHarOutputChannel,
}
app.Use(DisableRootStaticCache())
ginApp.Use(disableRootStaticCache())
if err := setUIMode(); err != nil {
logger.Log.Errorf("Error setting ui mode, err: %v", err)
}
app.Use(static.ServeRoot("/", "./site"))
app.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
api.WebSocketRoutes(app, &eventHandlers, startTime)
if config.Config.StandaloneMode {
routes.ConfigRoutes(app)
routes.UserRoutes(app)
routes.InstallRoutes(app)
staticFolder := "./site"
indexStaticFile := staticFolder + "/index.html"
if err := setUIFlags(indexStaticFile); err != nil {
logger.Log.Errorf("Error setting ui flags, err: %v", err)
}
routes.QueryRoutes(app)
routes.EntriesRoutes(app)
routes.MetadataRoutes(app)
routes.StatusRoutes(app)
routes.NotFoundRoute(app)
ginApp.Use(static.ServeRoot("/", staticFolder))
ginApp.NoRoute(func(c *gin.Context) {
c.File(indexStaticFile)
})
utils.StartServer(app)
ginApp.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if it's called before
api.WebSocketRoutes(ginApp, &eventHandlers)
if config.Config.OAS.Enable {
routes.OASRoutes(ginApp)
}
if config.Config.ServiceMap {
routes.ServiceMapRoutes(ginApp)
}
routes.QueryRoutes(ginApp)
routes.EntriesRoutes(ginApp)
routes.MetadataRoutes(ginApp)
routes.StatusRoutes(ginApp)
routes.DbRoutes(ginApp)
routes.ReplayRoutes(ginApp)
return ginApp
}
func DisableRootStaticCache() gin.HandlerFunc {
func runInApiServerMode(namespace string) *gin.Engine {
if err := config.LoadConfig(); err != nil {
logger.Log.Fatalf("Error loading config file %v", err)
}
app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort, config.Config.MaxDBSizeBytes, config.Config.LogLevel, config.Config.InsertionFilter)
api.StartResolving(namespace)
enableExpFeatureIfNeeded()
return hostApi(app.GetEntryInputChannel())
}
func runInTapperMode() {
logger.Log.Infof("Starting tapper, websocket address: %s", *apiServerAddress)
if *apiServerAddress == "" {
panic("API server address must be provided with --api-server-address when using --tap")
}
hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
tapOpts := &tap.TapOpts{
HostMode: hostMode,
}
filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
filteringOptions := getTrafficFilteringOptions()
tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, app.Extensions, filteringOptions)
socketConnection, err := dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay)
if err != nil {
panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
}
logger.Log.Infof("Connected successfully to websocket %s", *apiServerAddress)
go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
}
func runInStandaloneMode() {
api.StartResolving(*namespace)
outputItemsChannel := make(chan *tapApi.OutputChannelItem)
filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
filteringOptions := getTrafficFilteringOptions()
hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
tapOpts := &tap.TapOpts{HostMode: hostMode}
tap.StartPassiveTapper(tapOpts, outputItemsChannel, app.Extensions, filteringOptions)
go app.FilterItems(outputItemsChannel, filteredOutputItemsChannel)
go api.StartReadingEntries(filteredOutputItemsChannel, nil, app.ExtensionsMap)
ginApp := hostApi(nil)
utils.StartServer(ginApp)
}
func runInHarReaderMode() {
outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000)
filteredHarChannel := make(chan *tapApi.OutputChannelItem)
go app.FilterItems(outputItemsChannel, filteredHarChannel)
go api.StartReadingEntries(filteredHarChannel, harsDir, app.ExtensionsMap)
ginApp := hostApi(nil)
utils.StartServer(ginApp)
}
func enableExpFeatureIfNeeded() {
if config.Config.OAS.Enable {
oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
oasGenerator.Start()
}
if config.Config.ServiceMap {
serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMap)
serviceMapGenerator.Enable()
}
}
func disableRootStaticCache() gin.HandlerFunc {
return func(c *gin.Context) {
if c.Request.RequestURI == "/" {
// Disable cache only for the main static route
@ -268,13 +222,14 @@ func DisableRootStaticCache() gin.HandlerFunc {
}
}
func setUIMode() error {
func setUIFlags(uiIndexPath string) error {
read, err := ioutil.ReadFile(uiIndexPath)
if err != nil {
return err
}
replacedContent := strings.Replace(string(read), "__IS_STANDALONE__", strconv.FormatBool(config.Config.StandaloneMode), 1)
replacedContent := strings.Replace(string(read), "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS.Enable), 1)
replacedContent = strings.Replace(replacedContent, "__IS_SERVICE_MAP_ENABLED__", strconv.FormatBool(config.Config.ServiceMap), 1)
err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0)
if err != nil {
@ -284,28 +239,6 @@ func setUIMode() error {
return nil
}
func parseEnvVar(env string) map[string][]v1.Pod {
var mapOfList map[string][]v1.Pod
val, present := os.LookupEnv(env)
if !present {
return mapOfList
}
err := json.Unmarshal([]byte(val), &mapOfList)
if err != nil {
panic(fmt.Sprintf("env var %s's value of %v is invalid! must be map[string][]v1.Pod %v", env, mapOfList, err))
}
return mapOfList
}
func getTapTargets() []v1.Pod {
nodeName := os.Getenv(shared.NodeNameEnvVar)
tappedAddressesPerNodeDict := parseEnvVar(shared.TappedAddressesPerNodeDictEnvVar)
return tappedAddressesPerNodeDict[nodeName]
}
func getTrafficFilteringOptions() *tapApi.TrafficFilteringOptions {
filteringOptionsJson := os.Getenv(shared.MizuFilteringOptionsEnvVar)
if filteringOptionsJson == "" {
@ -322,16 +255,6 @@ func getTrafficFilteringOptions() *tapApi.TrafficFilteringOptions {
return &filteringOptions
}
func filterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem) {
for message := range inChannel {
if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
continue
}
outChannel <- message
}
}
func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tapApi.OutputChannelItem) {
if connection == nil {
panic("Websocket connection is nil")
@ -348,6 +271,10 @@ func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-cha
continue
}
if dbgctl.MizuTapperDisableSending {
continue
}
// NOTE: This is where the `*tapApi.OutputChannelItem` leaves the code
// and goes into the intermediate WebSocket.
err = connection.WriteMessage(websocket.TextMessage, marshaledData)
@ -367,21 +294,6 @@ func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-cha
}
}
func getSyncEntriesConfig() *shared.SyncEntriesConfig {
syncEntriesConfigJson := os.Getenv(shared.SyncEntriesConfigEnvVar)
if syncEntriesConfigJson == "" {
return nil
}
var syncEntriesConfig = &shared.SyncEntriesConfig{}
err := json.Unmarshal([]byte(syncEntriesConfigJson), syncEntriesConfig)
if err != nil {
panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.SyncEntriesConfig struct, err: %v", shared.SyncEntriesConfigEnvVar, syncEntriesConfigJson, err))
}
return syncEntriesConfig
}
func determineLogLevel() (logLevel logging.Level) {
logLevel, err := logging.LogLevel(os.Getenv(shared.LogLevelEnvVar))
if err != nil {
@ -400,6 +312,7 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time.
for i := 1; i < retryAmount; i++ {
socketConnection, _, err := dialer.Dial(socketAddress, nil)
if err != nil {
lastErr = err
if i < retryAmount {
logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay/time.Second)
time.Sleep(retryDelay)
@ -433,6 +346,14 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
} else {
tap.UpdateTapTargets(tapConfigMessage.TapTargets)
}
case shared.WebSocketMessageTypeUpdateTappedPods:
var tappedPodsMessage shared.WebSocketTappedPodsMessage
if err := json.Unmarshal(message, &tappedPodsMessage); err != nil {
logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
return
}
nodeName := os.Getenv(shared.NodeNameEnvVar)
tap.UpdateTapTargets(tappedPodsMessage.NodeToTappedPodMap[nodeName])
default:
logger.Log.Warningf("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
}
@ -440,3 +361,12 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
}
}
}
func initializeDependencies() {
dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} { return oas.GetDefaultOasGeneratorInstance(config.Config.OAS.MaxExampleLen) })
dependency.RegisterGenerator(dependency.EntriesInserter, func() interface{} { return api.GetBasenineEntryInserterInstance() })
dependency.RegisterGenerator(dependency.EntriesProvider, func() interface{} { return &entries.BasenineEntriesProvider{} })
dependency.RegisterGenerator(dependency.EntriesSocketStreamer, func() interface{} { return &api.BasenineEntryStreamer{} })
dependency.RegisterGenerator(dependency.EntryStreamerSocketConnector, func() interface{} { return &api.DefaultEntryStreamerSocketConnector{} })
}

View File

@ -1,107 +0,0 @@
package api
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"net/http"
"github.com/getkin/kin-openapi/openapi3"
"github.com/getkin/kin-openapi/openapi3filter"
"github.com/getkin/kin-openapi/routers"
legacyrouter "github.com/getkin/kin-openapi/routers/legacy"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
)
const (
ContractNotApplicable api.ContractStatus = 0
ContractPassed api.ContractStatus = 1
ContractFailed api.ContractStatus = 2
)
func loadOAS(ctx context.Context) (doc *openapi3.T, contractContent string, router routers.Router, err error) {
path := fmt.Sprintf("%s%s", shared.ConfigDirPath, shared.ContractFileName)
bytes, err := ioutil.ReadFile(path)
if err != nil {
return
}
contractContent = string(bytes)
loader := &openapi3.Loader{Context: ctx}
doc, _ = loader.LoadFromData(bytes)
err = doc.Validate(ctx)
if err != nil {
return
}
router, _ = legacyrouter.NewRouter(doc)
return
}
func validateOAS(ctx context.Context, doc *openapi3.T, router routers.Router, req *http.Request, res *http.Response) (isValid bool, reqErr error, resErr error) {
isValid = true
reqErr = nil
resErr = nil
// Find route
route, pathParams, err := router.FindRoute(req)
if err != nil {
return
}
// Validate request
requestValidationInput := &openapi3filter.RequestValidationInput{
Request: req,
PathParams: pathParams,
Route: route,
}
if reqErr = openapi3filter.ValidateRequest(ctx, requestValidationInput); reqErr != nil {
isValid = false
}
responseValidationInput := &openapi3filter.ResponseValidationInput{
RequestValidationInput: requestValidationInput,
Status: res.StatusCode,
Header: res.Header,
}
if res.Body != nil {
body, _ := ioutil.ReadAll(res.Body)
res.Body = ioutil.NopCloser(bytes.NewBuffer(body))
responseValidationInput.SetBodyBytes(body)
}
// Validate response.
if resErr = openapi3filter.ValidateResponse(ctx, responseValidationInput); resErr != nil {
isValid = false
}
return
}
func handleOAS(ctx context.Context, doc *openapi3.T, router routers.Router, req *http.Request, res *http.Response, contractContent string) (contract api.Contract) {
contract = api.Contract{
Content: contractContent,
Status: ContractNotApplicable,
}
isValid, reqErr, resErr := validateOAS(ctx, doc, router, req, res)
if isValid {
contract.Status = ContractPassed
} else {
contract.Status = ContractFailed
if reqErr != nil {
contract.RequestReason = reqErr.Error()
} else {
contract.RequestReason = ""
}
if resErr != nil {
contract.ResponseReason = resErr.Error()
} else {
contract.ResponseReason = ""
}
}
return
}

View File

@ -0,0 +1,71 @@
package api
import (
"fmt"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/agent/pkg/models"
tapApi "github.com/up9inc/mizu/tap/api"
)
type EntryStreamerSocketConnector interface {
SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error
SendMetadata(socketId int, metadata *basenine.Metadata) error
SendToastError(socketId int, err error) error
CleanupSocket(socketId int)
}
type DefaultEntryStreamerSocketConnector struct{}
func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error {
var message []byte
if params.EnableFullEntries {
message, _ = models.CreateFullEntryWebSocketMessage(entry)
} else {
protocol, ok := protocolsMap[entry.Protocol.ToString()]
if !ok {
return fmt.Errorf("protocol not found, protocol: %v", protocol)
}
extension, ok := extensionsMap[protocol.Name]
if !ok {
return fmt.Errorf("extension not found, extension: %v", protocol.Name)
}
base := extension.Dissector.Summarize(entry)
message, _ = models.CreateBaseEntryWebSocketMessage(base)
}
if err := SendToSocket(socketId, message); err != nil {
return err
}
return nil
}
func (e *DefaultEntryStreamerSocketConnector) SendMetadata(socketId int, metadata *basenine.Metadata) error {
metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
if err := SendToSocket(socketId, metadataBytes); err != nil {
return err
}
return nil
}
func (e *DefaultEntryStreamerSocketConnector) SendToastError(socketId int, err error) error {
toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
Type: "error",
AutoClose: 5000,
Text: fmt.Sprintf("Syntax error: %s", err.Error()),
})
if err := SendToSocket(socketId, toastBytes); err != nil {
return err
}
return nil
}
func (e *DefaultEntryStreamerSocketConnector) CleanupSocket(socketId int) {
socketObj := connectedWebsockets[socketId]
socketCleanup(socketId, socketObj)
}

View File

@ -5,24 +5,25 @@ import (
"context"
"encoding/json"
"fmt"
"mizuserver/pkg/holder"
"mizuserver/pkg/providers"
"os"
"path"
"sort"
"strings"
"time"
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/agent/pkg/dependency"
"github.com/up9inc/mizu/agent/pkg/oas"
"github.com/up9inc/mizu/agent/pkg/servicemap"
"github.com/up9inc/mizu/agent/pkg/har"
"github.com/up9inc/mizu/agent/pkg/holder"
"github.com/up9inc/mizu/agent/pkg/providers"
"github.com/up9inc/mizu/agent/pkg/resolver"
"github.com/up9inc/mizu/agent/pkg/utils"
"github.com/up9inc/mizu/logger"
tapApi "github.com/up9inc/mizu/tap/api"
"mizuserver/pkg/models"
"mizuserver/pkg/resolver"
"mizuserver/pkg/utils"
basenine "github.com/up9inc/basenine/client/go"
)
var k8sResolver *resolver.Resolver
@ -38,10 +39,8 @@ func StartResolving(namespace string) {
res.Start(ctx)
go func() {
for {
select {
case err := <-errOut:
logger.Log.Infof("name resolving error %s", err)
}
err := <-errOut
logger.Log.Infof("name resolving error %s", err)
}
}()
@ -63,7 +62,7 @@ func startReadingFiles(workingDir string) {
return
}
for true {
for {
dir, _ := os.Open(workingDir)
dirFiles, _ := dir.Readdir(-1)
@ -99,73 +98,69 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
panic("Channel of captured messages is nil")
}
connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
if err != nil {
panic(err)
}
connection.InsertMode()
disableOASValidation := false
ctx := context.Background()
doc, contractContent, router, err := loadOAS(ctx)
if err != nil {
logger.Log.Infof("Disabled OAS validation: %s", err.Error())
disableOASValidation = true
}
for item := range outputItems {
providers.EntryAdded()
extension := extensionsMap[item.Protocol.Name]
resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation)
if extension.Protocol.Name == "http" {
if !disableOASValidation {
var httpPair tapApi.HTTPRequestResponsePair
json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair)
resolvedSource, resolvedDestination, namespace := resolveIP(item.ConnectionInfo)
contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
mizuEntry.ContractStatus = contract.Status
mizuEntry.ContractRequestReason = contract.RequestReason
mizuEntry.ContractResponseReason = contract.ResponseReason
mizuEntry.ContractContent = contract.Content
}
harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
if err == nil {
rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name)
mizuEntry.Rules = rules
}
if namespace == "" && item.Namespace != tapApi.UnknownNamespace {
namespace = item.Namespace
}
mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestination, namespace)
data, err := json.Marshal(mizuEntry)
if err != nil {
panic(err)
logger.Log.Errorf("Error while marshaling entry: %v", err)
continue
}
connection.SendText(string(data))
entryInserter := dependency.GetInstance(dependency.EntriesInserter).(EntryInserter)
if err := entryInserter.Insert(mizuEntry); err != nil {
logger.Log.Errorf("Error inserting entry, err: %v", err)
}
summary := extension.Dissector.Summarize(mizuEntry)
providers.EntryAdded(len(data), summary)
serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMapSink)
serviceMapGenerator.NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGeneratorSink)
oasGenerator.HandleEntry(mizuEntry)
}
}
func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, resolvedDestination string) {
func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, resolvedDestination string, namespace string) {
if k8sResolver != nil {
unresolvedSource := connectionInfo.ClientIP
resolvedSource = k8sResolver.Resolve(unresolvedSource)
if resolvedSource == "" {
resolvedSourceObject := k8sResolver.Resolve(unresolvedSource)
if resolvedSourceObject == nil {
logger.Log.Debugf("Cannot find resolved name to source: %s", unresolvedSource)
if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" {
return
}
} else {
resolvedSource = resolvedSourceObject.FullAddress
namespace = resolvedSourceObject.Namespace
}
unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort)
resolvedDestination = k8sResolver.Resolve(unresolvedDestination)
if resolvedDestination == "" {
resolvedDestinationObject := k8sResolver.Resolve(unresolvedDestination)
if resolvedDestinationObject == nil {
logger.Log.Debugf("Cannot find resolved name to dest: %s", unresolvedDestination)
if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" {
return
}
} else {
resolvedDestination = resolvedDestinationObject.FullAddress
// Overwrite namespace (if it was set according to the source)
// Only overwrite if non-empty
if resolvedDestinationObject.Namespace != "" {
namespace = resolvedDestinationObject.Namespace
}
}
}
return resolvedSource, resolvedDestination
return resolvedSource, resolvedDestination, namespace
}
func CheckIsServiceIP(address string) bool {

View File

@ -0,0 +1,71 @@
package api
import (
"encoding/json"
"fmt"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/logger"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
"sync"
"time"
)
type EntryInserter interface {
Insert(entry *api.Entry) error
}
type BasenineEntryInserter struct {
connection *basenine.Connection
}
var instance *BasenineEntryInserter
var once sync.Once
func GetBasenineEntryInserterInstance() *BasenineEntryInserter {
once.Do(func() {
instance = &BasenineEntryInserter{}
})
return instance
}
func (e *BasenineEntryInserter) Insert(entry *api.Entry) error {
if e.connection == nil {
e.connection = initializeConnection()
}
data, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("error marshling entry, err: %v", err)
}
if err := e.connection.SendText(string(data)); err != nil {
e.connection.Close()
e.connection = nil
return fmt.Errorf("error sending text to database, err: %v", err)
}
return nil
}
func initializeConnection() *basenine.Connection{
for {
connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
if err != nil {
logger.Log.Errorf("Can't establish a new connection to Basenine server: %v", err)
time.Sleep(shared.BasenineReconnectInterval * time.Second)
continue
}
if err = connection.InsertMode(); err != nil {
logger.Log.Errorf("Insert mode call failed: %v", err)
connection.Close()
time.Sleep(shared.BasenineReconnectInterval * time.Second)
continue
}
return connection
}
}

View File

@ -0,0 +1,171 @@
package api
import (
"context"
"encoding/json"
"time"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/agent/pkg/dependency"
"github.com/up9inc/mizu/logger"
"github.com/up9inc/mizu/shared"
tapApi "github.com/up9inc/mizu/tap/api"
)
type EntryStreamer interface {
Get(ctx context.Context, socketId int, params *WebSocketParams) error
}
type BasenineEntryStreamer struct{}
func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *WebSocketParams) error {
var connection *basenine.Connection
entryStreamerSocketConnector := dependency.GetInstance(dependency.EntryStreamerSocketConnector).(EntryStreamerSocketConnector)
connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
if err != nil {
logger.Log.Errorf("Failed to establish a connection to Basenine: %v", err)
entryStreamerSocketConnector.CleanupSocket(socketId)
return err
}
data := make(chan []byte)
meta := make(chan []byte)
query := params.Query
if err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query); err != nil {
if err := entryStreamerSocketConnector.SendToastError(socketId, err); err != nil {
return err
}
entryStreamerSocketConnector.CleanupSocket(socketId)
return err
}
leftOff, err := e.fetch(socketId, params, entryStreamerSocketConnector)
if err != nil {
logger.Log.Errorf("Fetch error: %v", err)
}
handleDataChannel := func(c *basenine.Connection, data chan []byte) {
for {
bytes := <-data
if string(bytes) == basenine.CloseChannel {
return
}
var entry *tapApi.Entry
if err = json.Unmarshal(bytes, &entry); err != nil {
logger.Log.Debugf("Error unmarshalling entry: %v", err)
continue
}
if err := entryStreamerSocketConnector.SendEntry(socketId, entry, params); err != nil {
logger.Log.Errorf("Error sending entry to socket, err: %v", err)
return
}
}
}
handleMetaChannel := func(c *basenine.Connection, meta chan []byte) {
for {
bytes := <-meta
if string(bytes) == basenine.CloseChannel {
return
}
var metadata *basenine.Metadata
if err = json.Unmarshal(bytes, &metadata); err != nil {
logger.Log.Debugf("Error unmarshalling metadata: %v", err)
continue
}
if err := entryStreamerSocketConnector.SendMetadata(socketId, metadata); err != nil {
logger.Log.Errorf("Error sending metadata to socket, err: %v", err)
return
}
}
}
go handleDataChannel(connection, data)
go handleMetaChannel(connection, meta)
if err = connection.Query(leftOff, query, data, meta); err != nil {
logger.Log.Errorf("Query mode call failed: %v", err)
entryStreamerSocketConnector.CleanupSocket(socketId)
return err
}
go func() {
<-ctx.Done()
data <- []byte(basenine.CloseChannel)
meta <- []byte(basenine.CloseChannel)
connection.Close()
}()
return nil
}
// Reverses a []byte slice.
func (e *BasenineEntryStreamer) fetch(socketId int, params *WebSocketParams, connector EntryStreamerSocketConnector) (leftOff string, err error) {
if params.Fetch <= 0 {
leftOff = params.LeftOff
return
}
var data [][]byte
var firstMeta []byte
var lastMeta []byte
data, firstMeta, lastMeta, err = basenine.Fetch(
shared.BasenineHost,
shared.BaseninePort,
params.LeftOff,
-1,
params.Query,
params.Fetch,
time.Duration(params.TimeoutMs)*time.Millisecond,
)
if err != nil {
return
}
var firstMetadata *basenine.Metadata
if err = json.Unmarshal(firstMeta, &firstMetadata); err != nil {
return
}
leftOff = firstMetadata.LeftOff
var lastMetadata *basenine.Metadata
if err = json.Unmarshal(lastMeta, &lastMetadata); err != nil {
return
}
if err = connector.SendMetadata(socketId, lastMetadata); err != nil {
return
}
data = e.reverseBytesSlice(data)
for _, row := range data {
var entry *tapApi.Entry
if err = json.Unmarshal(row, &entry); err != nil {
break
}
if err = connector.SendEntry(socketId, entry, params); err != nil {
return
}
}
return
}
// Reverses a []byte slice.
func (e *BasenineEntryStreamer) reverseBytesSlice(arr [][]byte) (newArr [][]byte) {
for i := len(arr) - 1; i >= 0; i-- {
newArr = append(newArr, arr[i])
}
return newArr
}

View File

@ -1,28 +1,33 @@
package api
import (
"encoding/json"
"errors"
"fmt"
"mizuserver/pkg/middlewares"
"mizuserver/pkg/models"
"net/http"
"sync"
"time"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/debounce"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/agent/pkg/models"
"github.com/up9inc/mizu/agent/pkg/utils"
"github.com/up9inc/mizu/logger"
tapApi "github.com/up9inc/mizu/tap/api"
)
var (
extensionsMap map[string]*tapApi.Extension // global
protocolsMap map[string]*tapApi.Protocol //global
)
func InitMaps(extensions map[string]*tapApi.Extension, protocols map[string]*tapApi.Protocol) {
extensionsMap = extensions
protocolsMap = protocols
}
type EventHandlers interface {
WebSocketConnect(socketId int, isTapper bool)
WebSocketConnect(c *gin.Context, socketId int, isTapper bool)
WebSocketDisconnect(socketId int, isTapper bool)
WebSocketMessage(socketId int, message []byte)
WebSocketMessage(socketId int, isTapper bool, message []byte)
}
type SocketConnection struct {
@ -32,34 +37,54 @@ type SocketConnection struct {
isTapper bool
}
var websocketUpgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
type WebSocketParams struct {
LeftOff string `json:"leftOff"`
Query string `json:"query"`
EnableFullEntries bool `json:"enableFullEntries"`
Fetch int `json:"fetch"`
TimeoutMs int `json:"timeoutMs"`
}
var websocketIdsLock = sync.Mutex{}
var connectedWebsockets map[int]*SocketConnection
var connectedWebsocketIdCounter = 0
var (
websocketUpgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
}
websocketIdsLock = sync.Mutex{}
connectedWebsockets map[int]*SocketConnection
connectedWebsocketIdCounter = 0
SocketGetBrowserHandler gin.HandlerFunc
SocketGetTapperHandler gin.HandlerFunc
)
func init() {
websocketUpgrader.CheckOrigin = func(r *http.Request) bool { return true } // like cors for web socket
connectedWebsockets = make(map[int]*SocketConnection, 0)
connectedWebsockets = make(map[int]*SocketConnection)
}
func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) {
func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
SocketGetBrowserHandler = func(c *gin.Context) {
websocketHandler(c, eventHandlers, false)
}
SocketGetTapperHandler = func(c *gin.Context) {
websocketHandler(c, eventHandlers, true)
}
app.GET("/ws", func(c *gin.Context) {
websocketHandler(c.Writer, c.Request, eventHandlers, false, startTime)
}, middlewares.RequiresAuth())
SocketGetBrowserHandler(c)
})
app.GET("/wsTapper", func(c *gin.Context) { // TODO: add m2m authentication to this route
websocketHandler(c.Writer, c.Request, eventHandlers, true, startTime)
SocketGetTapperHandler(c)
})
}
func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool, startTime int64) {
ws, err := websocketUpgrader.Upgrade(w, r, nil)
func websocketHandler(c *gin.Context, eventHandlers EventHandlers, isTapper bool) {
ws, err := websocketUpgrader.Upgrade(c.Writer, c.Request, nil)
if err != nil {
logger.Log.Errorf("Failed to set websocket upgrade: %v", err)
logger.Log.Errorf("failed to set websocket upgrade: %v", err)
return
}
@ -71,110 +96,66 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
websocketIdsLock.Unlock()
var connection *basenine.Connection
var isQuerySet bool
// `!isTapper` means it's a connection from the web UI
if !isTapper {
connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
if err != nil {
panic(err)
}
}
data := make(chan []byte)
meta := make(chan []byte)
defer func() {
socketCleanup(socketId, connectedWebsockets[socketId])
data <- []byte(basenine.CloseChannel)
meta <- []byte(basenine.CloseChannel)
connection.Close()
if socketConnection := connectedWebsockets[socketId]; socketConnection != nil {
socketCleanup(socketId, socketConnection)
}
}()
eventHandlers.WebSocketConnect(socketId, isTapper)
eventHandlers.WebSocketConnect(c, socketId, isTapper)
startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(startTime)
SendToSocket(socketId, startTimeBytes)
startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(utils.StartTime)
if err = SendToSocket(socketId, startTimeBytes); err != nil {
logger.Log.Error(err)
}
for {
_, msg, err := ws.ReadMessage()
if err != nil {
if _, ok := err.(*websocket.CloseError); ok {
logger.Log.Debugf("Received websocket close message, socket id: %d", socketId)
logger.Log.Debugf("received websocket close message, socket id: %d", socketId)
} else {
logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
logger.Log.Errorf("error reading message, socket id: %d, error: %v", socketId, err)
}
break
}
if !isTapper && !isQuerySet {
query := string(msg)
err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
if err != nil {
toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
Type: "error",
AutoClose: 5000,
Text: fmt.Sprintf("Syntax error: %s", err.Error()),
})
SendToSocket(socketId, toastBytes)
break
}
isQuerySet = true
handleDataChannel := func(c *basenine.Connection, data chan []byte) {
for {
bytes := <-data
if string(bytes) == basenine.CloseChannel {
return
}
var entry *tapApi.Entry
err = json.Unmarshal(bytes, &entry)
base := tapApi.Summarize(entry)
baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
SendToSocket(socketId, baseEntryBytes)
}
}
handleMetaChannel := func(c *basenine.Connection, meta chan []byte) {
for {
bytes := <-meta
if string(bytes) == basenine.CloseChannel {
return
}
var metadata *basenine.Metadata
err = json.Unmarshal(bytes, &metadata)
if err != nil {
logger.Log.Debugf("Error recieving metadata: %v", err.Error())
}
metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
SendToSocket(socketId, metadataBytes)
}
}
go handleDataChannel(connection, data)
go handleMetaChannel(connection, meta)
connection.Query(query, data, meta)
} else {
eventHandlers.WebSocketMessage(socketId, msg)
}
eventHandlers.WebSocketMessage(socketId, isTapper, msg)
}
}
func SendToSocket(socketId int, message []byte) error {
socketObj := connectedWebsockets[socketId]
if socketObj == nil {
return fmt.Errorf("socket %v is disconnected", socketId)
}
socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
defer socketObj.lock.Unlock()
if connectedWebsockets[socketId] == nil {
return fmt.Errorf("socket %v is disconnected", socketId)
}
if err := socketObj.connection.SetWriteDeadline(time.Now().Add(time.Second * 10)); err != nil {
socketCleanup(socketId, socketObj)
return fmt.Errorf("error setting timeout to socket %v, err: %v", socketId, err)
}
if err := socketObj.connection.WriteMessage(websocket.TextMessage, message); err != nil {
socketCleanup(socketId, socketObj)
return fmt.Errorf("failed to write message to socket %v, err: %v", socketId, err)
}
return nil
}
func socketCleanup(socketId int, socketConnection *SocketConnection) {
err := socketConnection.connection.Close()
if err != nil {
logger.Log.Errorf("Error closing socket connection for socket id %d: %v", socketId, err)
logger.Log.Errorf("error closing socket connection for socket id %d: %v", socketId, err)
}
websocketIdsLock.Lock()
@ -183,29 +164,3 @@ func socketCleanup(socketId int, socketConnection *SocketConnection) {
socketConnection.eventHandlers.WebSocketDisconnect(socketId, socketConnection.isTapper)
}
var db = debounce.NewDebouncer(time.Second*5, func() {
logger.Log.Error("Successfully sent to socket")
})
func SendToSocket(socketId int, message []byte) error {
socketObj := connectedWebsockets[socketId]
if socketObj == nil {
return errors.New("Socket is disconnected")
}
var sent = false
time.AfterFunc(time.Second*5, func() {
if !sent {
logger.Log.Error("Socket timed out")
socketCleanup(socketId, socketObj)
}
})
socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
err := socketObj.connection.WriteMessage(1, message)
socketObj.lock.Unlock()
sent = true
return err
}

View File

@ -1,21 +1,28 @@
package api
import (
"context"
"encoding/json"
"fmt"
"mizuserver/pkg/models"
"mizuserver/pkg/providers"
"mizuserver/pkg/providers/tappersCount"
"mizuserver/pkg/up9"
"sync"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/agent/pkg/dependency"
"github.com/up9inc/mizu/agent/pkg/models"
"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
"github.com/up9inc/mizu/agent/pkg/providers/tappers"
tapApi "github.com/up9inc/mizu/tap/api"
"github.com/up9inc/mizu/logger"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
)
var browserClientSocketUUIDs = make([]int, 0)
type BrowserClient struct {
dataStreamCancelFunc context.CancelFunc
}
var browserClients = make(map[int]*BrowserClient, 0)
var tapperClientSocketUUIDs = make([]int, 0)
var socketListLock = sync.Mutex{}
type RoutesEventHandlers struct {
@ -23,46 +30,94 @@ type RoutesEventHandlers struct {
SocketOutChannel chan<- *tapApi.OutputChannelItem
}
func init() {
go up9.UpdateAnalyzeStatus(BroadcastToBrowserClients)
}
func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
func (h *RoutesEventHandlers) WebSocketConnect(_ *gin.Context, socketId int, isTapper bool) {
if isTapper {
logger.Log.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
tappersCount.Add()
tappers.Connected()
socketListLock.Lock()
tapperClientSocketUUIDs = append(tapperClientSocketUUIDs, socketId)
socketListLock.Unlock()
nodeToTappedPodMap := tappedPods.GetNodeToTappedPodMap()
SendTappedPods(socketId, nodeToTappedPodMap)
} else {
logger.Log.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
socketListLock.Lock()
browserClientSocketUUIDs = append(browserClientSocketUUIDs, socketId)
browserClients[socketId] = &BrowserClient{}
socketListLock.Unlock()
BroadcastTappedPodsStatus()
}
}
func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
if isTapper {
logger.Log.Infof("Websocket event - Tapper disconnected, socket ID: %d", socketId)
tappersCount.Remove()
tappers.Disconnected()
socketListLock.Lock()
removeSocketUUIDFromTapperSlice(socketId)
socketListLock.Unlock()
} else {
logger.Log.Infof("Websocket event - Browser socket disconnected, socket ID: %d", socketId)
socketListLock.Lock()
removeSocketUUIDFromBrowserSlice(socketId)
if browserClients[socketId] != nil && browserClients[socketId].dataStreamCancelFunc != nil {
browserClients[socketId].dataStreamCancelFunc()
}
delete(browserClients, socketId)
socketListLock.Unlock()
}
}
func BroadcastToBrowserClients(message []byte) {
for _, socketId := range browserClientSocketUUIDs {
for socketId := range browserClients {
go func(socketId int) {
err := SendToSocket(socketId, message)
if err != nil {
logger.Log.Errorf("error sending message to socket ID %d: %v", socketId, err)
if err := SendToSocket(socketId, message); err != nil {
logger.Log.Error(err)
}
}(socketId)
}
}
func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
func BroadcastToTapperClients(message []byte) {
for _, socketId := range tapperClientSocketUUIDs {
go func(socketId int) {
if err := SendToSocket(socketId, message); err != nil {
logger.Log.Error(err)
}
}(socketId)
}
}
func (h *RoutesEventHandlers) WebSocketMessage(socketId int, isTapper bool, message []byte) {
if isTapper {
HandleTapperIncomingMessage(message, h.SocketOutChannel, BroadcastToBrowserClients)
} else {
// we initiate the basenine stream after the first websocket message we receive (it contains the entry query), we then store a cancelfunc to later cancel this stream
if browserClients[socketId] != nil && browserClients[socketId].dataStreamCancelFunc == nil {
var params WebSocketParams
if err := json.Unmarshal(message, &params); err != nil {
logger.Log.Errorf("Error: %v", socketId, err)
return
}
entriesStreamer := dependency.GetInstance(dependency.EntriesSocketStreamer).(EntryStreamer)
ctx, cancelFunc := context.WithCancel(context.Background())
err := entriesStreamer.Get(ctx, socketId, &params)
if err != nil {
logger.Log.Errorf("error initializing basenine stream for browser socket %d %+v", socketId, err)
cancelFunc()
} else {
browserClients[socketId].dataStreamCancelFunc = cancelFunc
}
}
}
}
func HandleTapperIncomingMessage(message []byte, socketOutChannel chan<- *tapApi.OutputChannelItem, broadcastMessageFunc func([]byte)) {
var socketMessageBase shared.WebSocketMessageMetadata
err := json.Unmarshal(message, &socketMessageBase)
if err != nil {
@ -76,7 +131,7 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
} else {
// NOTE: This is where the message comes back from the intermediate WebSocket to code.
h.SocketOutChannel <- tappedEntryMessage.Data
socketOutChannel <- tappedEntryMessage.Data
}
case shared.WebSocketMessageTypeUpdateStatus:
var statusMessage shared.WebSocketStatusMessage
@ -84,15 +139,7 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
if err != nil {
logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
} else {
BroadcastToBrowserClients(message)
}
case shared.WebsocketMessageTypeOutboundLink:
var outboundLinkMessage models.WebsocketOutboundLinkMessage
err := json.Unmarshal(message, &outboundLinkMessage)
if err != nil {
logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
} else {
handleTLSLink(outboundLinkMessage)
broadcastMessageFunc(message)
}
default:
logger.Log.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
@ -100,35 +147,12 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
}
}
func handleTLSLink(outboundLinkMessage models.WebsocketOutboundLinkMessage) {
resolvedName := k8sResolver.Resolve(outboundLinkMessage.Data.DstIP)
if resolvedName != "" {
outboundLinkMessage.Data.DstIP = resolvedName
} else if outboundLinkMessage.Data.SuggestedResolvedName != "" {
outboundLinkMessage.Data.DstIP = outboundLinkMessage.Data.SuggestedResolvedName
}
cacheKey := fmt.Sprintf("%s -> %s:%d", outboundLinkMessage.Data.Src, outboundLinkMessage.Data.DstIP, outboundLinkMessage.Data.DstPort)
_, isInCache := providers.RecentTLSLinks.Get(cacheKey)
if isInCache {
return
} else {
providers.RecentTLSLinks.SetDefault(cacheKey, outboundLinkMessage.Data)
}
marshaledMessage, err := json.Marshal(outboundLinkMessage)
if err != nil {
logger.Log.Errorf("Error marshaling outbound link message for broadcasting: %v", err)
} else {
logger.Log.Errorf("Broadcasting outboundlink message %s", string(marshaledMessage))
BroadcastToBrowserClients(marshaledMessage)
}
}
func removeSocketUUIDFromBrowserSlice(uuidToRemove int) {
newUUIDSlice := make([]int, 0, len(browserClientSocketUUIDs))
for _, uuid := range browserClientSocketUUIDs {
func removeSocketUUIDFromTapperSlice(uuidToRemove int) {
newUUIDSlice := make([]int, 0, len(tapperClientSocketUUIDs))
for _, uuid := range tapperClientSocketUUIDs {
if uuid != uuidToRemove {
newUUIDSlice = append(newUUIDSlice, uuid)
}
}
browserClientSocketUUIDs = newUUIDSlice
tapperClientSocketUUIDs = newUUIDSlice
}

40
agent/pkg/api/utils.go Normal file
View File

@ -0,0 +1,40 @@
package api
import (
"encoding/json"
"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
"github.com/up9inc/mizu/logger"
"github.com/up9inc/mizu/shared"
)
func BroadcastTappedPodsStatus() {
tappedPodsStatus := tappedPods.GetTappedPodsStatus()
message := shared.CreateWebSocketStatusMessage(tappedPodsStatus)
if jsonBytes, err := json.Marshal(message); err != nil {
logger.Log.Errorf("Could not Marshal message %v", err)
} else {
BroadcastToBrowserClients(jsonBytes)
}
}
func SendTappedPods(socketId int, nodeToTappedPodMap shared.NodeToPodsMap) {
message := shared.CreateWebSocketTappedPodsMessage(nodeToTappedPodMap)
if jsonBytes, err := json.Marshal(message); err != nil {
logger.Log.Errorf("Could not Marshal message %v", err)
} else {
if err := SendToSocket(socketId, jsonBytes); err != nil {
logger.Log.Error(err)
}
}
}
func BroadcastTappedPodsToTappers(nodeToTappedPodMap shared.NodeToPodsMap) {
message := shared.CreateWebSocketTappedPodsMessage(nodeToTappedPodMap)
if jsonBytes, err := json.Marshal(message); err != nil {
logger.Log.Errorf("Could not Marshal message %v", err)
} else {
BroadcastToTapperClients(jsonBytes)
}
}

138
agent/pkg/app/main.go Normal file
View File

@ -0,0 +1,138 @@
package app
import (
"fmt"
"sort"
"time"
"github.com/antelman107/net-wait-go/wait"
"github.com/op/go-logging"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/agent/pkg/api"
"github.com/up9inc/mizu/agent/pkg/providers"
"github.com/up9inc/mizu/agent/pkg/utils"
"github.com/up9inc/mizu/logger"
tapApi "github.com/up9inc/mizu/tap/api"
"github.com/up9inc/mizu/tap/dbgctl"
amqpExt "github.com/up9inc/mizu/tap/extensions/amqp"
httpExt "github.com/up9inc/mizu/tap/extensions/http"
kafkaExt "github.com/up9inc/mizu/tap/extensions/kafka"
redisExt "github.com/up9inc/mizu/tap/extensions/redis"
)
var (
Extensions []*tapApi.Extension // global
ExtensionsMap map[string]*tapApi.Extension // global
ProtocolsMap map[string]*tapApi.Protocol //global
)
func LoadExtensions() {
Extensions = make([]*tapApi.Extension, 0)
ExtensionsMap = make(map[string]*tapApi.Extension)
ProtocolsMap = make(map[string]*tapApi.Protocol)
extensionHttp := &tapApi.Extension{}
dissectorHttp := httpExt.NewDissector()
dissectorHttp.Register(extensionHttp)
extensionHttp.Dissector = dissectorHttp
Extensions = append(Extensions, extensionHttp)
ExtensionsMap[extensionHttp.Protocol.Name] = extensionHttp
protocolsHttp := dissectorHttp.GetProtocols()
for k, v := range protocolsHttp {
ProtocolsMap[k] = v
}
if !dbgctl.MizuTapperDisableNonHttpExtensions {
extensionAmqp := &tapApi.Extension{}
dissectorAmqp := amqpExt.NewDissector()
dissectorAmqp.Register(extensionAmqp)
extensionAmqp.Dissector = dissectorAmqp
Extensions = append(Extensions, extensionAmqp)
ExtensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
protocolsAmqp := dissectorAmqp.GetProtocols()
for k, v := range protocolsAmqp {
ProtocolsMap[k] = v
}
extensionKafka := &tapApi.Extension{}
dissectorKafka := kafkaExt.NewDissector()
dissectorKafka.Register(extensionKafka)
extensionKafka.Dissector = dissectorKafka
Extensions = append(Extensions, extensionKafka)
ExtensionsMap[extensionKafka.Protocol.Name] = extensionKafka
protocolsKafka := dissectorKafka.GetProtocols()
for k, v := range protocolsKafka {
ProtocolsMap[k] = v
}
extensionRedis := &tapApi.Extension{}
dissectorRedis := redisExt.NewDissector()
dissectorRedis.Register(extensionRedis)
extensionRedis.Dissector = dissectorRedis
Extensions = append(Extensions, extensionRedis)
ExtensionsMap[extensionRedis.Protocol.Name] = extensionRedis
protocolsRedis := dissectorRedis.GetProtocols()
for k, v := range protocolsRedis {
ProtocolsMap[k] = v
}
}
sort.Slice(Extensions, func(i, j int) bool {
return Extensions[i].Protocol.Priority < Extensions[j].Protocol.Priority
})
api.InitMaps(ExtensionsMap, ProtocolsMap)
providers.InitProtocolToColor(ProtocolsMap)
}
func ConfigureBasenineServer(host string, port string, dbSize int64, logLevel logging.Level, insertionFilter string) {
if !wait.New(
wait.WithProto("tcp"),
wait.WithWait(200*time.Millisecond),
wait.WithBreak(50*time.Millisecond),
wait.WithDeadline(20*time.Second),
wait.WithDebug(logLevel == logging.DEBUG),
).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
logger.Log.Panicf("Basenine is not available!")
}
if err := basenine.Limit(host, port, dbSize); err != nil {
logger.Log.Panicf("Error while limiting database size: %v", err)
}
// Define the macros
for _, extension := range Extensions {
macros := extension.Dissector.Macros()
for macro, expanded := range macros {
if err := basenine.Macro(host, port, macro, expanded); err != nil {
logger.Log.Panicf("Error while adding a macro: %v", err)
}
}
}
// Set the insertion filter that comes from the config
if err := basenine.InsertionFilter(host, port, insertionFilter); err != nil {
logger.Log.Errorf("Error while setting the insertion filter: %v", err)
}
utils.StartTime = time.Now().UnixNano() / int64(time.Millisecond)
}
func GetEntryInputChannel() chan *tapApi.OutputChannelItem {
outputItemsChannel := make(chan *tapApi.OutputChannelItem)
filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
go FilterItems(outputItemsChannel, filteredOutputItemsChannel)
go api.StartReadingEntries(filteredOutputItemsChannel, nil, ExtensionsMap)
return outputItemsChannel
}
func FilterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem) {
for message := range inChannel {
if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
continue
}
outChannel <- message
}
}

View File

@ -1,154 +0,0 @@
package controllers
import (
"context"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/kubernetes"
"github.com/up9inc/mizu/shared/logger"
tapApi "github.com/up9inc/mizu/tap/api"
v1 "k8s.io/api/core/v1"
"mizuserver/pkg/config"
"mizuserver/pkg/models"
"mizuserver/pkg/providers"
"mizuserver/pkg/providers/tapConfig"
"mizuserver/pkg/providers/tappedPods"
"mizuserver/pkg/providers/tappersStatus"
"net/http"
"regexp"
"time"
)
var cancelTapperSyncer context.CancelFunc
func PostTapConfig(c *gin.Context) {
requestTapConfig := &models.TapConfig{}
if err := c.Bind(requestTapConfig); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
if cancelTapperSyncer != nil {
cancelTapperSyncer()
tappedPods.Set([]*shared.PodInfo{})
tappersStatus.Reset()
broadcastTappedPodsStatus()
}
var tappedNamespaces []string
for namespace, tapped := range requestTapConfig.TappedNamespaces {
if tapped {
tappedNamespaces = append(tappedNamespaces, namespace)
}
}
podRegex, _ := regexp.Compile(".*")
kubernetesProvider, err := providers.GetKubernetesProvider()
if err != nil {
c.JSON(http.StatusInternalServerError, err)
return
}
ctx, cancel := context.WithCancel(context.Background())
if _, err := startMizuTapperSyncer(ctx, kubernetesProvider, tappedNamespaces, *podRegex, []string{}, tapApi.TrafficFilteringOptions{}, false); err != nil {
c.JSON(http.StatusInternalServerError, err)
cancel()
return
}
cancelTapperSyncer = cancel
tapConfig.Save(requestTapConfig)
c.JSON(http.StatusOK, "OK")
}
func GetTapConfig(c *gin.Context) {
kubernetesProvider, err := providers.GetKubernetesProvider()
if err != nil {
c.JSON(http.StatusInternalServerError, err)
return
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
namespaces, err := kubernetesProvider.ListAllNamespaces(ctx)
if err != nil {
c.JSON(http.StatusInternalServerError, err)
return
}
savedTapConfig := tapConfig.Get()
tappedNamespaces := make(map[string]bool)
for _, namespace := range namespaces {
if namespace.Name == config.Config.MizuResourcesNamespace {
continue
}
tappedNamespaces[namespace.Name] = savedTapConfig.TappedNamespaces[namespace.Name]
}
tapConfigToReturn := models.TapConfig{TappedNamespaces: tappedNamespaces}
c.JSON(http.StatusOK, tapConfigToReturn)
}
func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, serviceMesh bool) (*kubernetes.MizuTapperSyncer, error) {
tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
TargetNamespaces: targetNamespaces,
PodFilterRegex: podFilterRegex,
MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
AgentImage: config.Config.AgentImage,
TapperResources: config.Config.TapperResources,
ImagePullPolicy: v1.PullPolicy(config.Config.PullPolicy),
LogLevel: config.Config.LogLevel,
IgnoredUserAgents: ignoredUserAgents,
MizuApiFilteringOptions: mizuApiFilteringOptions,
MizuServiceAccountExists: true, //assume service account exists since install mode will not function without it anyway
ServiceMesh: serviceMesh,
}, time.Now())
if err != nil {
return nil, err
}
// handle tapperSyncer events (pod changes and errors)
go func() {
for {
select {
case syncerErr, ok := <-tapperSyncer.ErrorOut:
if !ok {
logger.Log.Debug("mizuTapperSyncer err channel closed, ending listener loop")
return
}
logger.Log.Fatalf("fatal tap syncer error: %v", syncerErr)
case _, ok := <-tapperSyncer.TapPodChangesOut:
if !ok {
logger.Log.Debug("mizuTapperSyncer pod changes channel closed, ending listener loop")
return
}
tappedPods.Set(kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods))
broadcastTappedPodsStatus()
case tapperStatus, ok := <-tapperSyncer.TapperStatusChangedOut:
if !ok {
logger.Log.Debug("mizuTapperSyncer tapper status changed channel closed, ending listener loop")
return
}
tappersStatus.Set(&tapperStatus)
broadcastTappedPodsStatus()
case <-ctx.Done():
logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")
return
}
}
}()
return tapperSyncer, nil
}

View File

@ -0,0 +1,28 @@
package controllers
import (
"net/http"
"github.com/gin-gonic/gin"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/agent/pkg/app"
"github.com/up9inc/mizu/agent/pkg/config"
"github.com/up9inc/mizu/shared"
)
func Flush(c *gin.Context) {
if err := basenine.Flush(shared.BasenineHost, shared.BaseninePort); err != nil {
c.JSON(http.StatusBadRequest, err)
} else {
c.JSON(http.StatusOK, "Flushed.")
}
}
func Reset(c *gin.Context) {
if err := basenine.Reset(shared.BasenineHost, shared.BaseninePort); err != nil {
c.JSON(http.StatusBadRequest, err)
} else {
app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort, config.Config.MaxDBSizeBytes, config.Config.LogLevel, config.Config.InsertionFilter)
c.JSON(http.StatusOK, "Resetted.")
}
}

View File

@ -1,32 +1,22 @@
package controllers
import (
"encoding/json"
"mizuserver/pkg/models"
"mizuserver/pkg/utils"
"mizuserver/pkg/validation"
"net/http"
"strconv"
"time"
"github.com/up9inc/mizu/agent/pkg/dependency"
"github.com/up9inc/mizu/agent/pkg/entries"
"github.com/up9inc/mizu/agent/pkg/models"
"github.com/up9inc/mizu/agent/pkg/validation"
"github.com/gin-gonic/gin"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
tapApi "github.com/up9inc/mizu/tap/api"
"github.com/up9inc/mizu/logger"
)
var extensionsMap map[string]*tapApi.Extension // global
func InitExtensionsMap(ref map[string]*tapApi.Extension) {
extensionsMap = ref
}
func Error(c *gin.Context, err error) bool {
func HandleEntriesError(c *gin.Context, err error) bool {
if err != nil {
logger.Log.Errorf("Error getting entry: %v", err)
c.Error(err)
_ = c.Error(err)
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
"error": true,
"type": "error",
@ -53,44 +43,18 @@ func GetEntries(c *gin.Context) {
entriesRequest.TimeoutMs = 3000
}
data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
if err != nil {
c.JSON(http.StatusInternalServerError, validationError)
}
response := &models.EntriesResponse{}
var dataSlice []interface{}
for _, row := range data {
var entry *tapApi.Entry
err = json.Unmarshal(row, &entry)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{
"error": true,
"type": "error",
"autoClose": "5000",
"msg": string(row),
})
return // exit
entriesProvider := dependency.GetInstance(dependency.EntriesProvider).(entries.EntriesProvider)
entryWrappers, metadata, err := entriesProvider.GetEntries(entriesRequest)
if !HandleEntriesError(c, err) {
baseEntries := make([]interface{}, 0)
for _, entry := range entryWrappers {
baseEntries = append(baseEntries, entry.Base)
}
base := tapApi.Summarize(entry)
dataSlice = append(dataSlice, base)
c.JSON(http.StatusOK, models.EntriesResponse{
Data: baseEntries,
Meta: metadata,
})
}
var metadata *basenine.Metadata
err = json.Unmarshal(meta, &metadata)
if err != nil {
logger.Log.Debugf("Error recieving metadata: %v", err.Error())
}
response.Data = dataSlice
response.Meta = metadata
c.JSON(http.StatusOK, response)
}
func GetEntry(c *gin.Context) {
@ -104,42 +68,12 @@ func GetEntry(c *gin.Context) {
c.JSON(http.StatusBadRequest, validationError)
}
id, _ := strconv.Atoi(c.Param("id"))
var entry *tapApi.Entry
bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id, singleEntryRequest.Query)
if Error(c, err) {
return // exit
}
err = json.Unmarshal(bytes, &entry)
if err != nil {
c.JSON(http.StatusNotFound, gin.H{
"error": true,
"type": "error",
"autoClose": "5000",
"msg": string(bytes),
})
return // exit
}
id := c.Param("id")
extension := extensionsMap[entry.Protocol.Name]
representation, bodySize, _ := extension.Dissector.Represent(entry.Request, entry.Response)
entriesProvider := dependency.GetInstance(dependency.EntriesProvider).(entries.EntriesProvider)
entry, err := entriesProvider.GetEntry(singleEntryRequest, id)
var rules []map[string]interface{}
var isRulesEnabled bool
if entry.Protocol.Name == "http" {
harEntry, _ := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
isRulesEnabled = _isRulesEnabled
inrec, _ := json.Marshal(rulesMatched)
json.Unmarshal(inrec, &rules)
if !HandleEntriesError(c, err) {
c.JSON(http.StatusOK, entry)
}
c.JSON(http.StatusOK, tapApi.EntryWrapper{
Protocol: entry.Protocol,
Representation: string(representation),
BodySize: bodySize,
Data: entry,
Rules: rules,
IsRulesEnabled: isRulesEnabled,
})
}

View File

@ -1,18 +0,0 @@
package controllers
import (
"mizuserver/pkg/providers"
"net/http"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/shared/logger"
)
func IsSetupNecessary(c *gin.Context) {
if IsInstallNeeded, err := providers.IsInstallNeeded(); err != nil {
logger.Log.Errorf("unknown internal while checking if install is needed %s", err)
c.AbortWithStatusJSON(500, gin.H{"error": "internal error occured while checking if install is needed"})
} else {
c.JSON(http.StatusOK, IsInstallNeeded)
}
}

View File

@ -1,13 +1,14 @@
package controllers
import (
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/shared"
"mizuserver/pkg/version"
"net/http"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/agent/pkg/version"
"github.com/up9inc/mizu/shared"
)
func GetVersion(c *gin.Context) {
resp := shared.VersionResponse{SemVer: version.SemVer}
resp := shared.VersionResponse{Ver: version.Ver}
c.JSON(http.StatusOK, resp)
}

View File

@ -0,0 +1,68 @@
package controllers
import (
"net/http"
"github.com/chanced/openapi"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/agent/pkg/dependency"
"github.com/up9inc/mizu/agent/pkg/oas"
"github.com/up9inc/mizu/logger"
)
func GetOASServers(c *gin.Context) {
m := make([]string, 0)
oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
oasGenerator.GetServiceSpecs().Range(func(key, value interface{}) bool {
m = append(m, key.(string))
return true
})
c.JSON(http.StatusOK, m)
}
func GetOASSpec(c *gin.Context) {
oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
res, ok := oasGenerator.GetServiceSpecs().Load(c.Param("id"))
if !ok {
c.JSON(http.StatusNotFound, gin.H{
"error": true,
"type": "error",
"autoClose": "5000",
"msg": "Service not found among specs",
})
return // exit
}
gen := res.(*oas.SpecGen)
spec, err := gen.GetSpec()
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{
"error": true,
"type": "error",
"autoClose": "5000",
"msg": err,
})
return // exit
}
c.JSON(http.StatusOK, spec)
}
func GetOASAllSpecs(c *gin.Context) {
res := map[string]*openapi.OpenAPI{}
oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
oasGenerator.GetServiceSpecs().Range(func(key, value interface{}) bool {
svc := key.(string)
gen := value.(*oas.SpecGen)
spec, err := gen.GetSpec()
if err != nil {
logger.Log.Warningf("Failed to obtain spec for service %s: %s", svc, err)
return true
}
res[svc] = spec
return true
})
c.JSON(http.StatusOK, res)
}

View File

@ -0,0 +1,45 @@
package controllers
import (
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/agent/pkg/dependency"
"github.com/up9inc/mizu/agent/pkg/oas"
)
func TestGetOASServers(t *testing.T) {
recorder, c := getRecorderAndContext()
GetOASServers(c)
t.Logf("Written body: %s", recorder.Body.String())
}
func TestGetOASAllSpecs(t *testing.T) {
recorder, c := getRecorderAndContext()
GetOASAllSpecs(c)
t.Logf("Written body: %s", recorder.Body.String())
}
func TestGetOASSpec(t *testing.T) {
recorder, c := getRecorderAndContext()
c.Params = []gin.Param{{Key: "id", Value: "some"}}
GetOASSpec(c)
t.Logf("Written body: %s", recorder.Body.String())
}
func getRecorderAndContext() (*httptest.ResponseRecorder, *gin.Context) {
dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} {
return oas.GetDefaultOasGeneratorInstance(-1)
})
recorder := httptest.NewRecorder()
c, _ := gin.CreateTestContext(recorder)
oas.GetDefaultOasGeneratorInstance(-1).Start()
oas.GetDefaultOasGeneratorInstance(-1).GetServiceSpecs().Store("some", oas.NewGen("some"))
return recorder, c
}

View File

@ -0,0 +1,34 @@
package controllers
import (
"net/http"
"time"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/agent/pkg/replay"
"github.com/up9inc/mizu/agent/pkg/validation"
"github.com/up9inc/mizu/logger"
)
const (
replayTimeout = 10 * time.Second
)
func ReplayRequest(c *gin.Context) {
logger.Log.Debug("Starting replay")
replayDetails := &replay.Details{}
if err := c.Bind(replayDetails); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
logger.Log.Debugf("Validating replay, %v", replayDetails)
if err := validation.Validate(replayDetails); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
logger.Log.Debug("Executing replay, %v", replayDetails)
result := replay.ExecuteRequest(replayDetails, replayTimeout)
c.JSON(http.StatusOK, result)
}

View File

@ -0,0 +1,39 @@
package controllers
import (
"net/http"
"github.com/up9inc/mizu/agent/pkg/dependency"
"github.com/up9inc/mizu/agent/pkg/servicemap"
"github.com/gin-gonic/gin"
)
type ServiceMapController struct {
service servicemap.ServiceMap
}
func NewServiceMapController() *ServiceMapController {
serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMap)
return &ServiceMapController{
service: serviceMapGenerator,
}
}
func (s *ServiceMapController) Status(c *gin.Context) {
c.JSON(http.StatusOK, s.service.GetStatus())
}
func (s *ServiceMapController) Get(c *gin.Context) {
response := &servicemap.ServiceMapResponse{
Status: s.service.GetStatus(),
Nodes: s.service.GetNodes(),
Edges: s.service.GetEdges(),
}
c.JSON(http.StatusOK, response)
}
func (s *ServiceMapController) Reset(c *gin.Context) {
s.service.Reset()
s.Status(c)
}

View File

@ -0,0 +1,151 @@
package controllers
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/up9inc/mizu/agent/pkg/dependency"
"github.com/up9inc/mizu/agent/pkg/servicemap"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
tapApi "github.com/up9inc/mizu/tap/api"
)
const (
a = "aService"
b = "bService"
Ip = "127.0.0.1"
Port = "80"
)
var (
TCPEntryA = &tapApi.TCP{
Name: a,
Port: Port,
IP: fmt.Sprintf("%s.%s", Ip, a),
}
TCPEntryB = &tapApi.TCP{
Name: b,
Port: Port,
IP: fmt.Sprintf("%s.%s", Ip, b),
}
)
var ProtocolHttp = &tapApi.Protocol{
ProtocolSummary: tapApi.ProtocolSummary{
Name: "http",
Version: "1.1",
Abbreviation: "HTTP",
},
LongName: "Hypertext Transfer Protocol -- HTTP/1.1",
Macro: "http",
BackgroundColor: "#205cf5",
ForegroundColor: "#ffffff",
FontSize: 12,
ReferenceLink: "https://datatracker.ietf.org/doc/html/rfc2616",
Ports: []string{"80", "443", "8080"},
Priority: 0,
}
type ServiceMapControllerSuite struct {
suite.Suite
c *ServiceMapController
w *httptest.ResponseRecorder
g *gin.Context
}
func (s *ServiceMapControllerSuite) SetupTest() {
dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
s.c = NewServiceMapController()
s.c.service.Enable()
s.c.service.(servicemap.ServiceMapSink).NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
s.w = httptest.NewRecorder()
s.g, _ = gin.CreateTestContext(s.w)
}
func (s *ServiceMapControllerSuite) TestGetStatus() {
assert := s.Assert()
s.c.Status(s.g)
assert.Equal(http.StatusOK, s.w.Code)
var status servicemap.ServiceMapStatus
err := json.Unmarshal(s.w.Body.Bytes(), &status)
assert.NoError(err)
assert.Equal("enabled", status.Status)
assert.Equal(1, status.EntriesProcessedCount)
assert.Equal(2, status.NodeCount)
assert.Equal(1, status.EdgeCount)
}
func (s *ServiceMapControllerSuite) TestGet() {
assert := s.Assert()
s.c.Get(s.g)
assert.Equal(http.StatusOK, s.w.Code)
var response servicemap.ServiceMapResponse
err := json.Unmarshal(s.w.Body.Bytes(), &response)
assert.NoError(err)
// response status
assert.Equal("enabled", response.Status.Status)
assert.Equal(1, response.Status.EntriesProcessedCount)
assert.Equal(2, response.Status.NodeCount)
assert.Equal(1, response.Status.EdgeCount)
// response nodes
aNode := servicemap.ServiceMapNode{
Id: 1,
Name: TCPEntryA.Name,
Entry: TCPEntryA,
Resolved: true,
Count: 1,
}
bNode := servicemap.ServiceMapNode{
Id: 2,
Name: TCPEntryB.Name,
Entry: TCPEntryB,
Resolved: true,
Count: 1,
}
assert.Contains(response.Nodes, aNode)
assert.Contains(response.Nodes, bNode)
assert.Len(response.Nodes, 2)
// response edges
assert.Equal([]servicemap.ServiceMapEdge{
{
Source: aNode,
Destination: bNode,
Protocol: ProtocolHttp,
Count: 1,
},
}, response.Edges)
}
func (s *ServiceMapControllerSuite) TestGetReset() {
assert := s.Assert()
s.c.Reset(s.g)
assert.Equal(http.StatusOK, s.w.Code)
var status servicemap.ServiceMapStatus
err := json.Unmarshal(s.w.Body.Bytes(), &status)
assert.NoError(err)
assert.Equal("enabled", status.Status)
assert.Equal(0, status.EntriesProcessedCount)
assert.Equal(0, status.NodeCount)
assert.Equal(0, status.EdgeCount)
}
func TestServiceMapControllerSuite(t *testing.T) {
suite.Run(t, new(ServiceMapControllerSuite))
}

View File

@ -1,56 +1,55 @@
package controllers
import (
"encoding/json"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
"mizuserver/pkg/api"
"mizuserver/pkg/holder"
"mizuserver/pkg/providers"
"mizuserver/pkg/providers/tappedPods"
"mizuserver/pkg/providers/tappersCount"
"mizuserver/pkg/providers/tappersStatus"
"mizuserver/pkg/up9"
"mizuserver/pkg/validation"
"fmt"
"net/http"
"strconv"
"time"
core "k8s.io/api/core/v1"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/agent/pkg/api"
"github.com/up9inc/mizu/agent/pkg/holder"
"github.com/up9inc/mizu/agent/pkg/providers"
"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
"github.com/up9inc/mizu/agent/pkg/providers/tappers"
"github.com/up9inc/mizu/agent/pkg/validation"
"github.com/up9inc/mizu/logger"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/kubernetes"
)
func HealthCheck(c *gin.Context) {
tappers := make([]*shared.TapperStatus, 0)
for _, value := range tappersStatus.Get() {
tappers = append(tappers, value)
tappersStatus := make([]*shared.TapperStatus, 0)
for _, value := range tappers.GetStatus() {
tappersStatus = append(tappersStatus, value)
}
response := shared.HealthResponse{
TappedPods: tappedPods.Get(),
TappersCount: tappersCount.Get(),
TappersStatus: tappers,
TappedPods: tappedPods.Get(),
ConnectedTappersCount: tappers.GetConnectedCount(),
TappersStatus: tappersStatus,
}
c.JSON(http.StatusOK, response)
}
func PostTappedPods(c *gin.Context) {
var requestTappedPods []*shared.PodInfo
var requestTappedPods []core.Pod
if err := c.Bind(&requestTappedPods); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
podInfos := kubernetes.GetPodInfosForPods(requestTappedPods)
logger.Log.Infof("[Status] POST request: %d tapped pods", len(requestTappedPods))
tappedPods.Set(requestTappedPods)
broadcastTappedPodsStatus()
}
tappedPods.Set(podInfos)
api.BroadcastTappedPodsStatus()
func broadcastTappedPodsStatus() {
tappedPodsStatus := tappedPods.GetTappedPodsStatus()
message := shared.CreateWebSocketStatusMessage(tappedPodsStatus)
if jsonBytes, err := json.Marshal(message); err != nil {
logger.Log.Errorf("Could not Marshal message %v", err)
} else {
api.BroadcastToBrowserClients(jsonBytes)
}
nodeToTappedPodMap := kubernetes.GetNodeHostToTappedPodsMap(requestTappedPods)
tappedPods.SetNodeToTappedPodMap(nodeToTappedPodMap)
api.BroadcastTappedPodsToTappers(nodeToTappedPodMap)
}
func PostTapperStatus(c *gin.Context) {
@ -66,22 +65,12 @@ func PostTapperStatus(c *gin.Context) {
}
logger.Log.Infof("[Status] POST request, tapper status: %v", tapperStatus)
tappersStatus.Set(tapperStatus)
broadcastTappedPodsStatus()
tappers.SetStatus(tapperStatus)
api.BroadcastTappedPodsStatus()
}
func GetTappersCount(c *gin.Context) {
c.JSON(http.StatusOK, tappersCount.Get())
}
func GetAuthStatus(c *gin.Context) {
authStatus, err := providers.GetAuthStatus()
if err != nil {
c.JSON(http.StatusInternalServerError, err)
return
}
c.JSON(http.StatusOK, authStatus)
func GetConnectedTappersCount(c *gin.Context) {
c.JSON(http.StatusOK, tappers.GetConnectedCount())
}
func GetTappingStatus(c *gin.Context) {
@ -89,16 +78,29 @@ func GetTappingStatus(c *gin.Context) {
c.JSON(http.StatusOK, tappedPodsStatus)
}
func AnalyzeInformation(c *gin.Context) {
c.JSON(http.StatusOK, up9.GetAnalyzeInfo())
}
func GetGeneralStats(c *gin.Context) {
c.JSON(http.StatusOK, providers.GetGeneralStats())
}
func GetRecentTLSLinks(c *gin.Context) {
c.JSON(http.StatusOK, providers.GetAllRecentTLSAddresses())
func GetTrafficStats(c *gin.Context) {
startTime, endTime, err := getStartEndTime(c)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, providers.GetTrafficStats(startTime, endTime))
}
func getStartEndTime(c *gin.Context) (time.Time, time.Time, error) {
startTimeValue, err := strconv.Atoi(c.Query("startTimeMs"))
if err != nil {
return time.UnixMilli(0), time.UnixMilli(0), fmt.Errorf("invalid start time: %v", err)
}
endTimeValue, err := strconv.Atoi(c.Query("endTimeMs"))
if err != nil {
return time.UnixMilli(0), time.UnixMilli(0), fmt.Errorf("invalid end time: %v", err)
}
return time.UnixMilli(int64(startTimeValue)), time.UnixMilli(int64(endTimeValue)), nil
}
func GetCurrentResolvingInformation(c *gin.Context) {

View File

@ -1,39 +0,0 @@
package controllers
import (
"mizuserver/pkg/providers"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/shared/logger"
)
func Login(c *gin.Context) {
if token, err := providers.PerformLogin(c.PostForm("username"), c.PostForm("password"), c.Request.Context()); err != nil {
c.AbortWithStatusJSON(401, gin.H{"error": "bad login"})
} else {
c.JSON(200, gin.H{"token": token})
}
}
func Logout(c *gin.Context) {
token := c.GetHeader("x-session-token")
if err := providers.Logout(token, c.Request.Context()); err != nil {
c.AbortWithStatusJSON(500, gin.H{"error": "error occured while logging out, the session might still be valid"})
} else {
c.JSON(200, "")
}
}
func Register(c *gin.Context) {
if token, _, err, formErrorMessages := providers.RegisterUser(c.PostForm("username"), c.PostForm("password"), c.Request.Context()); err != nil {
if formErrorMessages != nil {
logger.Log.Infof("user attempted to register but had form errors %v %v", formErrorMessages, err)
c.AbortWithStatusJSON(400, formErrorMessages)
} else {
logger.Log.Errorf("unknown internal error registering user %s", err)
c.AbortWithStatusJSON(500, gin.H{"error": "internal error occured while registering"})
}
} else {
c.JSON(200, gin.H{"token": token})
}
}

View File

@ -0,0 +1,11 @@
package dependency
var typeInitializerMap = make(map[ContainerType]func() interface{}, 0)
func RegisterGenerator(name ContainerType, fn func() interface{}) {
typeInitializerMap[name] = fn
}
func GetInstance(name ContainerType) interface{} {
return typeInitializerMap[name]()
}

View File

@ -0,0 +1,12 @@
package dependency
type ContainerType string
const (
ServiceMapGeneratorDependency ContainerType = "ServiceMapGeneratorDependency"
OasGeneratorDependency ContainerType = "OasGeneratorDependency"
EntriesInserter ContainerType = "EntriesInserter"
EntriesProvider ContainerType = "EntriesProvider"
EntriesSocketStreamer ContainerType = "EntriesSocketStreamer"
EntryStreamerSocketConnector ContainerType = "EntryStreamerSocketConnector"
)

View File

@ -0,0 +1,103 @@
package entries
import (
"encoding/json"
"errors"
"fmt"
"time"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/agent/pkg/app"
"github.com/up9inc/mizu/agent/pkg/models"
"github.com/up9inc/mizu/logger"
"github.com/up9inc/mizu/shared"
tapApi "github.com/up9inc/mizu/tap/api"
)
type EntriesProvider interface {
GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error)
GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId string) (*tapApi.EntryWrapper, error)
}
type BasenineEntriesProvider struct{}
func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error) {
data, _, lastMeta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
if err != nil {
return nil, nil, err
}
var dataSlice []*tapApi.EntryWrapper
for _, row := range data {
var entry *tapApi.Entry
err = json.Unmarshal(row, &entry)
if err != nil {
return nil, nil, err
}
protocol, ok := app.ProtocolsMap[entry.Protocol.ToString()]
if !ok {
return nil, nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
}
extension, ok := app.ExtensionsMap[protocol.Name]
if !ok {
return nil, nil, fmt.Errorf("extension not found, extension: %v", protocol.Name)
}
base := extension.Dissector.Summarize(entry)
dataSlice = append(dataSlice, &tapApi.EntryWrapper{
Protocol: *protocol,
Data: entry,
Base: base,
})
}
var metadata *basenine.Metadata
err = json.Unmarshal(lastMeta, &metadata)
if err != nil {
logger.Log.Debugf("Error recieving metadata: %v", err.Error())
}
return dataSlice, metadata, nil
}
func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId string) (*tapApi.EntryWrapper, error) {
var entry *tapApi.Entry
bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, entryId, singleEntryRequest.Query)
if err != nil {
return nil, err
}
err = json.Unmarshal(bytes, &entry)
if err != nil {
return nil, errors.New(string(bytes))
}
protocol, ok := app.ProtocolsMap[entry.Protocol.ToString()]
if !ok {
return nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
}
extension, ok := app.ExtensionsMap[protocol.Name]
if !ok {
return nil, fmt.Errorf("extension not found, extension: %v", protocol.Name)
}
base := extension.Dissector.Summarize(entry)
var representation []byte
representation, err = extension.Dissector.Represent(entry.Request, entry.Response)
if err != nil {
return nil, err
}
return &tapApi.EntryWrapper{
Protocol: *protocol,
Representation: string(representation),
Data: entry,
Base: base,
}, nil
}

376
agent/pkg/har/types.go Normal file
View File

@ -0,0 +1,376 @@
package har
import (
"encoding/base64"
"time"
"unicode/utf8"
"github.com/up9inc/mizu/logger"
)
/*
HTTP Archive (HAR) format
https://w3c.github.io/web-performance/specs/HAR/Overview.html
*/
// HAR is a container type for deserialization
type HAR struct {
Log Log `json:"log"`
}
// Log represents the root of the exported data. This object MUST be present and its name MUST be "log".
type Log struct {
// The object contains the following name/value pairs:
// Required. Version number of the format.
Version string `json:"version"`
// Required. An object of type creator that contains the name and version
// information of the log creator application.
Creator Creator `json:"creator"`
// Optional. An object of type browser that contains the name and version
// information of the user agent.
Browser Browser `json:"browser"`
// Optional. An array of objects of type page, each representing one exported
// (tracked) page. Leave out this field if the application does not support
// grouping by pages.
Pages []Page `json:"pages,omitempty"`
// Required. An array of objects of type entry, each representing one
// exported (tracked) HTTP request.
Entries []Entry `json:"entries"`
// Optional. A comment provided by the user or the application. Sorting
// entries by startedDateTime (starting from the oldest) is preferred way how
// to export data since it can make importing faster. However the reader
// application should always make sure the array is sorted (if required for
// the import).
Comment string `json:"comment"`
}
// Creator contains information about the log creator application
type Creator struct {
// Required. The name of the application that created the log.
Name string `json:"name"`
// Required. The version number of the application that created the log.
Version string `json:"version"`
// Optional. A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// Browser that created the log
type Browser struct {
// Required. The name of the browser that created the log.
Name string `json:"name"`
// Required. The version number of the browser that created the log.
Version string `json:"version"`
// Optional. A comment provided by the user or the browser.
Comment string `json:"comment"`
}
// Page object for every exported web page and one <entry> object for every HTTP request.
// In case when an HTTP trace tool isn't able to group requests by a page,
// the <pages> object is empty and individual requests doesn't have a parent page.
type Page struct {
/* There is one <page> object for every exported web page and one <entry>
object for every HTTP request. In case when an HTTP trace tool isn't able to
group requests by a page, the <pages> object is empty and individual
requests doesn't have a parent page.
*/
// Date and time stamp for the beginning of the page load
// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00).
StartedDateTime string `json:"startedDateTime"`
// Unique identifier of a page within the . Entries use it to refer the parent page.
ID string `json:"id"`
// Page title.
Title string `json:"title"`
// Detailed timing info about page load.
PageTiming PageTiming `json:"pageTiming"`
// (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// PageTiming describes timings for various events (states) fired during the page load.
// All times are specified in milliseconds. If a time info is not available appropriate field is set to -1.
type PageTiming struct {
// Content of the page loaded. Number of milliseconds since page load started
// (page.startedDateTime). Use -1 if the timing does not apply to the current
// request.
// Depeding on the browser, onContentLoad property represents DOMContentLoad
// event or document.readyState == interactive.
OnContentLoad int `json:"onContentLoad"`
// Page is loaded (onLoad event fired). Number of milliseconds since page
// load started (page.startedDateTime). Use -1 if the timing does not apply
// to the current request.
OnLoad int `json:"onLoad"`
// (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment"`
}
// Entry is a unique, optional Reference to the parent page.
// Leave out this field if the application does not support grouping by pages.
type Entry struct {
Pageref string `json:"pageref,omitempty"`
// Date and time stamp of the request start
// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD).
StartedDateTime string `json:"startedDateTime"`
// Total elapsed time of the request in milliseconds. This is the sum of all
// timings available in the timings object (i.e. not including -1 values) .
Time int `json:"time"`
// Detailed info about the request.
Request Request `json:"request"`
// Detailed info about the response.
Response Response `json:"response"`
// Info about cache usage.
Cache Cache `json:"cache"`
// Detailed timing info about request/response round trip.
PageTimings PageTimings `json:"pageTimings"`
// optional (new in 1.2) IP address of the server that was connected
// (result of DNS resolution).
ServerIPAddress string `json:"serverIPAddress,omitempty"`
// optional (new in 1.2) Unique ID of the parent TCP/IP connection, can be
// the client port number. Note that a port number doesn't have to be unique
// identifier in cases where the port is shared for more connections. If the
// port isn't available for the application, any other unique connection ID
// can be used instead (e.g. connection index). Leave out this field if the
// application doesn't support this info.
Connection string `json:"connection,omitempty"`
// (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// Request contains detailed info about performed request.
type Request struct {
// Request method (GET, POST, ...).
Method string `json:"method"`
// Absolute URL of the request (fragments are not included).
URL string `json:"url"`
// Request HTTP Version.
HTTPVersion string `json:"httpVersion"`
// List of cookie objects.
Cookies []Cookie `json:"cookies"`
// List of header objects.
Headers []NVP `json:"headers"`
// List of query parameter objects.
QueryString []NVP `json:"queryString"`
// Posted data.
PostData PostData `json:"postData"`
// Total number of bytes from the start of the HTTP request message until
// (and including) the double CRLF before the body. Set to -1 if the info
// is not available.
HeaderSize int `json:"headerSize"`
// Size of the request body (POST data payload) in bytes. Set to -1 if the
// info is not available.
BodySize int `json:"bodySize"`
// (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment"`
}
// Response contains detailed info about the response.
type Response struct {
// Response status.
Status int `json:"status"`
// Response status description.
StatusText string `json:"statusText"`
// Response HTTP Version.
HTTPVersion string `json:"httpVersion"`
// List of cookie objects.
Cookies []Cookie `json:"cookies"`
// List of header objects.
Headers []NVP `json:"headers"`
// Details about the response body.
Content Content `json:"content"`
// Redirection target URL from the Location response header.
RedirectURL string `json:"redirectURL"`
// Total number of bytes from the start of the HTTP response message until
// (and including) the double CRLF before the body. Set to -1 if the info is
// not available.
// The size of received response-headers is computed only from headers that
// are really received from the server. Additional headers appended by the
// browser are not included in this number, but they appear in the list of
// header objects.
HeadersSize int `json:"headersSize"`
// Size of the received response body in bytes. Set to zero in case of
// responses coming from the cache (304). Set to -1 if the info is not
// available.
BodySize int `json:"bodySize"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// Cookie contains list of all cookies (used in <request> and <response> objects).
type Cookie struct {
// The name of the cookie.
Name string `json:"name"`
// The cookie value.
Value string `json:"value"`
// optional The path pertaining to the cookie.
Path string `json:"path,omitempty"`
// optional The host of the cookie.
Domain string `json:"domain,omitempty"`
// optional Cookie expiration time.
// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.123+02:00).
Expires string `json:"expires,omitempty"`
// optional Set to true if the cookie is HTTP only, false otherwise.
HTTPOnly bool `json:"httpOnly,omitempty"`
// optional (new in 1.2) True if the cookie was transmitted over ssl, false
// otherwise.
Secure bool `json:"secure,omitempty"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// NVP is simply a name/value pair with a comment
type NVP struct {
Name string `json:"name"`
Value string `json:"value"`
Comment string `json:"comment,omitempty"`
}
// PostData describes posted data, if any (embedded in <request> object).
type PostData struct {
// Mime type of posted data.
MimeType string `json:"mimeType"`
// List of posted parameters (in case of URL encoded parameters).
Params []PostParam `json:"params"`
// Plain text posted data
Text string `json:"text"`
// optional (new in 1.2) A comment provided by the user or the
// application.
Comment string `json:"comment,omitempty"`
}
func (d PostData) B64Decoded() (bool, []byte, string) {
// there is a weird gap in HAR spec 1.2, that does not define encoding for binary POST bodies
// we have own convention of putting `base64` into comment field to handle it similar to response `Content`
return b64Decoded(d.Comment, d.Text)
}
// PostParam is a list of posted parameters, if any (embedded in <postData> object).
type PostParam struct {
// name of a posted parameter.
Name string `json:"name"`
// optional value of a posted parameter or content of a posted file.
Value string `json:"value,omitempty"`
// optional name of a posted file.
FileName string `json:"fileName,omitempty"`
// optional content type of a posted file.
ContentType string `json:"contentType,omitempty"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// Content describes details about response content (embedded in <response> object).
type Content struct {
// Length of the returned content in bytes. Should be equal to
// response.bodySize if there is no compression and bigger when the content
// has been compressed.
Size int `json:"size"`
// optional Number of bytes saved. Leave out this field if the information
// is not available.
Compression int `json:"compression,omitempty"`
// MIME type of the response text (value of the Content-Type response
// header). The charset attribute of the MIME type is included (if
// available).
MimeType string `json:"mimeType"`
// optional Response body sent from the server or loaded from the browser
// cache. This field is populated with textual content only. The text field
// is either HTTP decoded text or a encoded (e.g. "base64") representation of
// the response body. Leave out this field if the information is not
// available.
Text string `json:"text,omitempty"`
// optional (new in 1.2) Encoding used for response text field e.g
// "base64". Leave out this field if the text field is HTTP decoded
// (decompressed & unchunked), than trans-coded from its original character
// set into UTF-8.
Encoding string `json:"encoding,omitempty"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
func (c Content) B64Decoded() (bool, []byte, string) {
return b64Decoded(c.Encoding, c.Text)
}
func b64Decoded(enc string, text string) (isBinary bool, asBytes []byte, asString string) {
if enc == "base64" {
decoded, err := base64.StdEncoding.DecodeString(text)
if err != nil {
logger.Log.Warningf("Failed to decode content as base64: %s", text)
return false, []byte(text), text
}
valid := utf8.Valid(decoded)
return !valid, decoded, string(decoded)
} else {
return false, nil, text
}
}
// Cache contains info about a request coming from browser cache.
type Cache struct {
// optional State of a cache entry before the request. Leave out this field
// if the information is not available.
BeforeRequest CacheObject `json:"beforeRequest,omitempty"`
// optional State of a cache entry after the request. Leave out this field if
// the information is not available.
AfterRequest CacheObject `json:"afterRequest,omitempty"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// CacheObject is used by both beforeRequest and afterRequest
type CacheObject struct {
// optional - Expiration time of the cache entry.
Expires string `json:"expires,omitempty"`
// The last time the cache entry was opened.
LastAccess string `json:"lastAccess"`
// Etag
ETag string `json:"eTag"`
// The number of times the cache entry has been opened.
HitCount int `json:"hitCount"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// PageTimings describes various phases within request-response round trip.
// All times are specified in milliseconds.
type PageTimings struct {
Blocked int `json:"blocked,omitempty"`
// optional - Time spent in a queue waiting for a network connection. Use -1
// if the timing does not apply to the current request.
DNS int `json:"dns,omitempty"`
// optional - DNS resolution time. The time required to resolve a host name.
// Use -1 if the timing does not apply to the current request.
Connect int `json:"connect,omitempty"`
// optional - Time required to create TCP connection. Use -1 if the timing
// does not apply to the current request.
Send int `json:"send"`
// Time required to send HTTP request to the server.
Wait int `json:"wait"`
// Waiting for a response from the server.
Receive int `json:"receive"`
// Time required to read entire response from the server (or cache).
Ssl int `json:"ssl,omitempty"`
// optional (new in 1.2) - Time required for SSL/TLS negotiation. If this
// field is defined then the time is also included in the connect field (to
// ensure backward compatibility with HAR 1.1). Use -1 if the timing does not
// apply to the current request.
Comment string `json:"comment,omitempty"`
// optional (new in 1.2) - A comment provided by the user or the application.
}
// TestResult contains results for an individual HTTP request
type TestResult struct {
URL string `json:"url"`
Status int `json:"status"` // 200, 500, etc.
StartTime time.Time `json:"startTime"`
EndTime time.Time `json:"endTime"`
Latency int `json:"latency"` // milliseconds
Method string `json:"method"`
HarFile string `json:"harfile"`
}
// aliases for martian lib compatibility
type Header = NVP
type QueryString = NVP
type Param = PostParam
type Timings = PageTimings

View File

@ -0,0 +1,38 @@
package har
import "testing"
func TestContentEncoded(t *testing.T) {
testCases := []struct {
text string
isBinary bool
expectedStr string
binaryLen int
}{
{"not-base64", false, "not-base64", 10},
{"dGVzdA==", false, "test", 4},
{"test", true, "\f@A", 3}, // valid UTF-8 with some non-printable chars
{"IsDggPCAgPiAgID8gICAgN/vv/e/v/u/v7/9v7+/vyIKIu+3kO+3ke+3ku+3k++3lO+3le+3lu+3l++3mO+3me+3mu+3m++3nO+3ne+3nu+3n++3oO+3oe+3ou+3o++3pO+3pe+3pu+3p++3qO+3qe+3qu+3q++3rO+3re+3ru+3ryIK", true, "test", 132}, // invalid UTF-8 (thus binary), taken from https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
}
for _, tc := range testCases {
c := Content{
Encoding: "base64",
Text: tc.text,
}
isBinary, asBytes, asString := c.B64Decoded()
_ = asBytes
if tc.isBinary != isBinary {
t.Errorf("Binary flag mismatch: %t != %t", tc.isBinary, isBinary)
}
if !isBinary && tc.expectedStr != asString {
t.Errorf("Decode value mismatch: %s != %s", tc.expectedStr, asString)
}
if tc.binaryLen != len(asBytes) {
t.Errorf("Binary len mismatch: %d != %d", tc.binaryLen, len(asBytes))
}
}
}

215
agent/pkg/har/utils.go Normal file
View File

@ -0,0 +1,215 @@
package har
import (
"bytes"
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/up9inc/mizu/logger"
)
func BuildHeaders(rawHeaders map[string]interface{}) ([]Header, string, string, string, string, string) {
var host, scheme, authority, path, status string
headers := make([]Header, 0, len(rawHeaders))
for key, value := range rawHeaders {
headers = append(headers, Header{
Name: key,
Value: value.(string),
})
if key == "Host" {
host = value.(string)
}
if key == ":authority" {
authority = value.(string)
}
if key == ":scheme" {
scheme = value.(string)
}
if key == ":path" {
path = value.(string)
}
if key == ":status" {
status = value.(string)
}
}
return headers, host, scheme, authority, path, status
}
func BuildPostParams(rawParams []interface{}) []Param {
params := make([]Param, 0, len(rawParams))
for _, param := range rawParams {
p := param.(map[string]interface{})
name := ""
if p["name"] != nil {
name = p["name"].(string)
}
value := ""
if p["value"] != nil {
value = p["value"].(string)
}
fileName := ""
if p["fileName"] != nil {
fileName = p["fileName"].(string)
}
contentType := ""
if p["contentType"] != nil {
contentType = p["contentType"].(string)
}
params = append(params, Param{
Name: name,
Value: value,
FileName: fileName,
ContentType: contentType,
})
}
return params
}
func NewRequest(request map[string]interface{}) (harRequest *Request, err error) {
headers, host, scheme, authority, path, _ := BuildHeaders(request["headers"].(map[string]interface{}))
cookies := make([]Cookie, 0)
postData, _ := request["postData"].(map[string]interface{})
mimeType := postData["mimeType"]
if mimeType == nil {
mimeType = ""
}
text := postData["text"]
postDataText := ""
if text != nil {
postDataText = text.(string)
}
queryString := make([]QueryString, 0)
for key, value := range request["queryString"].(map[string]interface{}) {
if valuesInterface, ok := value.([]interface{}); ok {
for _, valueInterface := range valuesInterface {
queryString = append(queryString, QueryString{
Name: key,
Value: valueInterface.(string),
})
}
} else {
queryString = append(queryString, QueryString{
Name: key,
Value: value.(string),
})
}
}
url := fmt.Sprintf("http://%s%s", host, request["url"].(string))
if strings.HasPrefix(mimeType.(string), "application/grpc") {
url = fmt.Sprintf("%s://%s%s", scheme, authority, path)
}
harParams := make([]Param, 0)
if postData["params"] != nil {
harParams = BuildPostParams(postData["params"].([]interface{}))
}
harRequest = &Request{
Method: request["method"].(string),
URL: url,
HTTPVersion: request["httpVersion"].(string),
HeaderSize: -1,
BodySize: bytes.NewBufferString(postDataText).Len(),
QueryString: queryString,
Headers: headers,
Cookies: cookies,
PostData: PostData{
MimeType: mimeType.(string),
Params: harParams,
Text: postDataText,
},
}
return
}
func NewResponse(response map[string]interface{}) (harResponse *Response, err error) {
headers, _, _, _, _, _status := BuildHeaders(response["headers"].(map[string]interface{}))
cookies := make([]Cookie, 0)
content, _ := response["content"].(map[string]interface{})
mimeType := content["mimeType"]
if mimeType == nil {
mimeType = ""
}
encoding := content["encoding"]
text := content["text"]
bodyText := ""
if text != nil {
bodyText = text.(string)
}
harContent := &Content{
Encoding: encoding.(string),
MimeType: mimeType.(string),
Text: bodyText,
Size: len(bodyText),
}
status := int(response["status"].(float64))
if strings.HasPrefix(mimeType.(string), "application/grpc") {
if _status != "" {
status, err = strconv.Atoi(_status)
}
if err != nil {
logger.Log.Errorf("Failed converting status to int %s (%v,%+v)", err, err, err)
return nil, errors.New("failed converting response status to int for HAR")
}
}
harResponse = &Response{
HTTPVersion: response["httpVersion"].(string),
Status: status,
StatusText: response["statusText"].(string),
HeadersSize: -1,
BodySize: bytes.NewBufferString(bodyText).Len(),
Headers: headers,
Cookies: cookies,
Content: *harContent,
}
return
}
func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*Entry, error) {
harRequest, err := NewRequest(request)
if err != nil {
logger.Log.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err)
return nil, errors.New("failed converting request to HAR")
}
harResponse, err := NewResponse(response)
if err != nil {
logger.Log.Errorf("Failed converting response to HAR %s (%v,%+v)", err, err, err)
return nil, errors.New("failed converting response to HAR")
}
if elapsedTime < 1 {
elapsedTime = 1
}
harEntry := Entry{
StartedDateTime: startTime.Format(time.RFC3339),
Time: int(elapsedTime),
Request: *harRequest,
Response: *harResponse,
Cache: Cache{},
PageTimings: PageTimings{
Send: -1,
Wait: -1,
Receive: int(elapsedTime),
},
}
return &harEntry, nil
}

View File

@ -1,6 +1,6 @@
package holder
import "mizuserver/pkg/resolver"
import "github.com/up9inc/mizu/agent/pkg/resolver"
var k8sResolver *resolver.Resolver
@ -11,4 +11,3 @@ func SetResolver(param *resolver.Resolver) {
func GetResolver() *resolver.Resolver {
return k8sResolver
}

View File

@ -7,7 +7,7 @@ func CORSMiddleware() gin.HandlerFunc {
c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With, x-session-token")
c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT")
c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE")
if c.Request.Method == "OPTIONS" {
c.AbortWithStatus(204)

View File

@ -1,49 +0,0 @@
package middlewares
import (
"mizuserver/pkg/config"
"mizuserver/pkg/providers"
"time"
"github.com/gin-gonic/gin"
"github.com/patrickmn/go-cache"
"github.com/up9inc/mizu/shared/logger"
)
const cachedValidTokensRetainmentTime = time.Minute * 1
var cachedValidTokens = cache.New(cachedValidTokensRetainmentTime, cachedValidTokensRetainmentTime)
func RequiresAuth() gin.HandlerFunc {
return func(c *gin.Context) {
// auth is irrelevant for ephermeral mizu
if !config.Config.StandaloneMode {
c.Next()
return
}
token := c.GetHeader("x-session-token")
if token == "" {
c.AbortWithStatusJSON(401, gin.H{"error": "token header is empty"})
return
}
if _, isTokenCached := cachedValidTokens.Get(token); isTokenCached {
c.Next()
return
}
if isTokenValid, err := providers.VerifyToken(token, c.Request.Context()); err != nil {
logger.Log.Errorf("error verifying token %s", err)
c.AbortWithStatusJSON(401, gin.H{"error": "unknown auth error occured"})
return
} else if !isTokenValid {
c.AbortWithStatusJSON(401, gin.H{"error": "invalid token"})
return
}
cachedValidTokens.Set(token, true, cachedValidTokensRetainmentTime)
c.Next()
}
}

View File

@ -2,26 +2,16 @@ package models
import (
"encoding/json"
"mizuserver/pkg/rules"
"github.com/up9inc/mizu/agent/pkg/har"
tapApi "github.com/up9inc/mizu/tap/api"
"github.com/google/martian/har"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
)
func GetEntry(r *tapApi.Entry, v tapApi.DataUnmarshaler) error {
return v.UnmarshalData(r)
}
type TapConfig struct {
TappedNamespaces map[string]bool `json:"tappedNamespaces"`
}
type EntriesRequest struct {
LeftOff int `form:"leftOff" validate:"required,min=-1"`
LeftOff string `form:"leftOff" validate:"required"`
Direction int `form:"direction" validate:"required,oneof='1' '-1'"`
Query string `form:"query"`
Limit int `form:"limit" validate:"required,min=1"`
@ -42,21 +32,16 @@ type WebSocketEntryMessage struct {
Data *tapApi.BaseEntry `json:"data,omitempty"`
}
type WebSocketFullEntryMessage struct {
*shared.WebSocketMessageMetadata
Data *tapApi.Entry `json:"data,omitempty"`
}
type WebSocketTappedEntryMessage struct {
*shared.WebSocketMessageMetadata
Data *tapApi.OutputChannelItem
}
type WebsocketOutboundLinkMessage struct {
*shared.WebSocketMessageMetadata
Data *tap.OutboundLink
}
type AuthStatus struct {
Email string `json:"email"`
Model string `json:"model"`
}
type ToastMessage struct {
Type string `json:"type"`
AutoClose uint `json:"autoClose"`
@ -88,20 +73,20 @@ func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntry) ([]byte, error) {
return json.Marshal(message)
}
func CreateWebsocketTappedEntryMessage(base *tapApi.OutputChannelItem) ([]byte, error) {
message := &WebSocketTappedEntryMessage{
func CreateFullEntryWebSocketMessage(entry *tapApi.Entry) ([]byte, error) {
message := &WebSocketFullEntryMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
MessageType: shared.WebSocketMessageTypeTappedEntry,
MessageType: shared.WebSocketMessageTypeFullEntry,
},
Data: base,
Data: entry,
}
return json.Marshal(message)
}
func CreateWebsocketOutboundLinkMessage(base *tap.OutboundLink) ([]byte, error) {
message := &WebsocketOutboundLinkMessage{
func CreateWebsocketTappedEntryMessage(base *tapApi.OutputChannelItem) ([]byte, error) {
message := &WebSocketTappedEntryMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
MessageType: shared.WebsocketMessageTypeOutboundLink,
MessageType: shared.WebSocketMessageTypeTappedEntry,
},
Data: base,
}
@ -157,13 +142,3 @@ type ExtendedCreator struct {
*har.Creator
Source *string `json:"_source"`
}
func RunValidationRulesState(harEntry har.Entry, service string) (tapApi.ApplicableRules, []rules.RulesMatched, bool) {
resultPolicyToSend, isEnabled := rules.MatchRequestPolicy(harEntry, service)
statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend)
return tapApi.ApplicableRules{Status: statusPolicyToSend, Latency: latency, NumberOfRules: numberOfRules}, resultPolicyToSend, isEnabled
}
type InstallState struct {
Completed bool `json:"completed"`
}

119
agent/pkg/oas/counters.go Normal file
View File

@ -0,0 +1,119 @@
package oas
import (
"fmt"
"github.com/chanced/openapi"
"math"
"strings"
)
type Counter struct {
Entries int `json:"entries"`
Failures int `json:"failures"`
FirstSeen float64 `json:"firstSeen"`
LastSeen float64 `json:"lastSeen"`
SumRT float64 `json:"sumRT"`
SumDuration float64 `json:"sumDuration"`
}
func (c *Counter) addEntry(ts float64, rt float64, succ bool, dur float64) {
if dur < 0 {
panic("Duration cannot be negative")
}
c.Entries += 1
c.SumRT += rt
c.SumDuration += dur
if !succ {
c.Failures += 1
}
if c.FirstSeen == 0 {
c.FirstSeen = ts
} else {
c.FirstSeen = math.Min(c.FirstSeen, ts)
}
c.LastSeen = math.Max(c.LastSeen, ts)
}
func (c *Counter) addOther(other *Counter) {
c.Entries += other.Entries
c.SumRT += other.SumRT
c.Failures += other.Failures
c.SumDuration += other.SumDuration
if c.FirstSeen == 0 {
c.FirstSeen = other.FirstSeen
} else {
c.FirstSeen = math.Min(c.FirstSeen, other.FirstSeen)
}
c.LastSeen = math.Max(c.LastSeen, other.LastSeen)
}
type CounterMap map[string]*Counter
func (m *CounterMap) addOther(other *CounterMap) {
for src, cmap := range *other {
if existing, ok := (*m)[src]; ok {
existing.addOther(cmap)
} else {
copied := *cmap
(*m)[src] = &copied
}
}
}
func setCounterMsgIfOk(oldStr string, cnt *Counter) string {
tpl := "Mizu observed %d entries (%d failed), at %.3f hits/s, average response time is %.3f seconds"
if oldStr == "" || (strings.HasPrefix(oldStr, "Mizu ") && strings.HasSuffix(oldStr, " seconds")) {
return fmt.Sprintf(tpl, cnt.Entries, cnt.Failures, cnt.SumDuration/float64(cnt.Entries), cnt.SumRT/float64(cnt.Entries))
}
return oldStr
}
type CounterMaps struct {
counterTotal Counter
counterMapTotal CounterMap
}
func (m *CounterMaps) processOp(opObj *openapi.Operation) error {
if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
counter := new(Counter)
err := opObj.Extensions.DecodeExtension(CountersTotal, counter)
if err != nil {
return err
}
m.counterTotal.addOther(counter)
opObj.Description = setCounterMsgIfOk(opObj.Description, counter)
}
if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
counterMap := new(CounterMap)
err := opObj.Extensions.DecodeExtension(CountersPerSource, counterMap)
if err != nil {
return err
}
m.counterMapTotal.addOther(counterMap)
}
return nil
}
func (m *CounterMaps) processOas(oas *openapi.OpenAPI) error {
if oas.Extensions == nil {
oas.Extensions = openapi.Extensions{}
}
err := oas.Extensions.SetExtension(CountersTotal, m.counterTotal)
if err != nil {
return err
}
err = oas.Extensions.SetExtension(CountersPerSource, m.counterMapTotal)
if err != nil {
return nil
}
return nil
}

View File

@ -0,0 +1,212 @@
package oas
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"github.com/up9inc/mizu/agent/pkg/har"
"github.com/up9inc/mizu/logger"
)
func getFiles(baseDir string) (result []string, err error) {
result = make([]string, 0)
logger.Log.Infof("Reading files from tree: %s", baseDir)
inputs := []string{baseDir}
// https://yourbasic.org/golang/list-files-in-directory/
visitor := func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 {
path, _ = os.Readlink(path)
inputs = append(inputs, path)
return nil
}
ext := strings.ToLower(filepath.Ext(path))
if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
result = append(result, path)
}
return nil
}
for len(inputs) > 0 {
path := inputs[0]
inputs = inputs[1:]
err = filepath.Walk(path, visitor)
}
sort.SliceStable(result, func(i, j int) bool {
return fileSize(result[i]) < fileSize(result[j])
})
logger.Log.Infof("Got files: %d", len(result))
return result, err
}
func fileSize(fname string) int64 {
fi, err := os.Stat(fname)
if err != nil {
panic(err)
}
return fi.Size()
}
func feedEntries(fromFiles []string, isSync bool, gen *defaultOasGenerator) (count uint, err error) {
badFiles := make([]string, 0)
cnt := uint(0)
for _, file := range fromFiles {
logger.Log.Info("Processing file: " + file)
ext := strings.ToLower(filepath.Ext(file))
eCnt := uint(0)
switch ext {
case ".har":
eCnt, err = feedFromHAR(file, isSync, gen)
if err != nil {
logger.Log.Warning("Failed processing file: " + err.Error())
badFiles = append(badFiles, file)
continue
}
case ".ldjson":
eCnt, err = feedFromLDJSON(file, isSync, gen)
if err != nil {
logger.Log.Warning("Failed processing file: " + err.Error())
badFiles = append(badFiles, file)
continue
}
default:
return 0, errors.New("Unsupported file extension: " + ext)
}
cnt += eCnt
}
for _, f := range badFiles {
logger.Log.Infof("Bad file: %s", f)
}
return cnt, nil
}
func feedFromHAR(file string, isSync bool, gen *defaultOasGenerator) (uint, error) {
fd, err := os.Open(file)
if err != nil {
panic(err)
}
defer fd.Close()
data, err := ioutil.ReadAll(fd)
if err != nil {
return 0, err
}
var harDoc har.HAR
err = json.Unmarshal(data, &harDoc)
if err != nil {
return 0, err
}
cnt := uint(0)
for _, entry := range harDoc.Log.Entries {
cnt += 1
feedEntry(&entry, "", file, gen, fmt.Sprintf("%024d", cnt))
}
return cnt, nil
}
func feedEntry(entry *har.Entry, source string, file string, gen *defaultOasGenerator, cnt string) {
entry.Comment = file
if entry.Response.Status == 302 {
logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
}
if strings.Contains(entry.Request.URL, "some") { // for debugging
logger.Log.Debugf("Interesting: %s", entry.Request.URL)
}
u, err := url.Parse(entry.Request.URL)
if err != nil {
logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
}
ews := EntryWithSource{Entry: *entry, Source: source, Destination: u.Host, Id: cnt}
gen.handleHARWithSource(&ews)
}
func feedFromLDJSON(file string, isSync bool, gen *defaultOasGenerator) (uint, error) {
fd, err := os.Open(file)
if err != nil {
panic(err)
}
defer fd.Close()
reader := bufio.NewReader(fd)
var meta map[string]interface{}
buf := strings.Builder{}
cnt := uint(0)
source := ""
for {
substr, isPrefix, err := reader.ReadLine()
if err == io.EOF {
break
}
buf.WriteString(string(substr))
if isPrefix {
continue
}
line := buf.String()
buf.Reset()
if meta == nil {
err := json.Unmarshal([]byte(line), &meta)
if err != nil {
return 0, err
}
if s, ok := meta["_source"]; ok && s != nil {
source = s.(string)
}
} else {
var entry har.Entry
err := json.Unmarshal([]byte(line), &entry)
if err != nil {
logger.Log.Warningf("Failed decoding entry: %s", line)
} else {
cnt += 1
feedEntry(&entry, source, file, gen, fmt.Sprintf("%024d", cnt))
}
}
}
return cnt, nil
}
func TestFilesList(t *testing.T) {
res, err := getFiles("./test_artifacts/")
t.Log(len(res))
t.Log(res)
if err != nil || len(res) != 3 {
t.Logf("Should return 2 files but returned %d", len(res))
t.FailNow()
}
}

185
agent/pkg/oas/gibberish.go Normal file
View File

@ -0,0 +1,185 @@
package oas
import (
"math"
"regexp"
"strings"
"unicode"
)
var (
patUuid4 = regexp.MustCompile(`(?i)[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`)
patEmail = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`)
patLongNum = regexp.MustCompile(`^\d{3,}$`)
patLongNumB = regexp.MustCompile(`[^\d]\d{3,}`)
patLongNumA = regexp.MustCompile(`\d{3,}[^\d]`)
)
func IsGibberish(str string) bool {
if IsVersionString(str) {
return false
}
if patEmail.MatchString(str) {
return true
}
if patUuid4.MatchString(str) {
return true
}
if patLongNum.MatchString(str) || patLongNumB.MatchString(str) || patLongNumA.MatchString(str) {
return true
}
//alNum := cleanStr(str, isAlNumRune)
//alpha := cleanStr(str, isAlphaRune)
// noiseAll := isNoisy(alNum)
//triAll := isTrigramBad(strings.ToLower(alpha))
// _ = noiseAll
isNotAlNum := func(r rune) bool { return !isAlNumRune(r) }
chunks := strings.FieldsFunc(str, isNotAlNum)
noisyLen := 0
alnumLen := 0
for _, chunk := range chunks {
alnumLen += len(chunk)
noise := isNoisy(chunk)
tri := isTrigramBad(strings.ToLower(chunk))
if noise || tri {
noisyLen += len(chunk)
}
}
return float64(noisyLen) > 0
//if float64(noisyLen) > 0 {
// return true
//}
//if len(chunks) > 0 && float64(noisyLen) >= float64(alnumLen)/3.0 {
// return true
//}
//if triAll {
//return true
//}
// return false
}
func noiseLevel(str string) (score float64) {
// opinionated algo of certain char pairs marking the non-human strings
prev := *new(rune)
cnt := 0.0
for _, char := range str {
cnt += 1
if prev > 0 {
switch {
// continued class of upper/lower/digit adds no noise
case unicode.IsUpper(prev) && unicode.IsUpper(char):
case unicode.IsLower(prev) && unicode.IsLower(char):
case unicode.IsDigit(prev) && unicode.IsDigit(char):
// upper =>
case unicode.IsUpper(prev) && unicode.IsLower(char):
score += 0.10
case unicode.IsUpper(prev) && unicode.IsDigit(char):
score += 0.5
// lower =>
case unicode.IsLower(prev) && unicode.IsUpper(char):
score += 0.75
case unicode.IsLower(prev) && unicode.IsDigit(char):
score += 0.5
// digit =>
case unicode.IsDigit(prev) && unicode.IsUpper(char):
score += 0.75
case unicode.IsDigit(prev) && unicode.IsLower(char):
score += 1.0
// the rest is 100% noise
default:
score += 1
}
}
prev = char
}
return score
}
func IsVersionString(component string) bool {
if component == "" {
return false
}
hasV := false
if strings.HasPrefix(component, "v") {
component = component[1:]
hasV = true
}
for _, c := range component {
if string(c) != "." && !unicode.IsDigit(c) {
return false
}
}
if !hasV && !strings.Contains(component, ".") {
return false
}
return true
}
func trigramScore(str string) (float64, int) {
tgScore := 0.0
trigrams := ngrams(str, 3)
if len(trigrams) > 0 {
for _, trigram := range trigrams {
score, found := corpus_trigrams[trigram]
if found {
tgScore += score
}
}
}
return tgScore, len(trigrams)
}
func isTrigramBad(s string) bool {
tgScore, cnt := trigramScore(s)
if cnt > 0 {
val := math.Sqrt(tgScore) / float64(cnt)
val2 := tgScore / float64(cnt)
threshold := 0.005
bad := val < threshold
threshold2 := math.Log(float64(cnt)-2) * 0.1
bad2 := val2 < threshold2
return bad && bad2 // TODO: improve this logic to be clearer
}
return false
}
func isNoisy(s string) bool {
noise := noiseLevel(s)
if len(s) > 0 {
val := (noise * noise) / float64(len(s))
threshold := 0.6
bad := val > threshold
return bad
}
return false
}
func ngrams(s string, n int) []string {
result := make([]string, 0)
for i := 0; i < len(s)-n+1; i++ {
result = append(result, s[i:i+n])
}
return result
}

View File

@ -0,0 +1,212 @@
package oas
import (
"testing"
)
func TestNegative(t *testing.T) {
cases := []string{
"",
"{}",
"0.0.29",
"0.1",
"1.0",
"1.0.0",
"2.1.73",
"abTestV2",
"actionText,setName,setAttribute,save,ignore,onEnd,getContext,end,get",
"AddUserGroupLink",
"advert-management.adBlockerMessage.html",
"agents.author.1.json",
"animated-gif",
"b", // can be valid hexadecimal
"big-danger-coronavirus-panic-greater-crisis",
"breakout-box",
"callback",
"core.algorithm_execution.view",
"core.devices.view",
"data.json",
"dialog.overlay.infinity.json",
"domain-input",
"embeddable", // lul, it's a valid HEX!
"embeddable_blip",
"E PLURIBUS UNUM",
"etc",
"eu-central-1a",
"fcgi-bin",
"footer.include.html",
"fullHashes:find",
"generate-feed",
"GetAds",
"GetCart",
"GetUniversalVariableUser",
"github-audit-exports",
"g.js",
"g.pixel",
".html",
"Hugo Michiels",
"image.sbix",
"index.html",
"iPad",
"Joanna Mazewski",
"LibGit2Sharp",
"Michael_Vaughan1.png",
"New RSS feed has been generated",
"nick-clegg",
"opt-out",
"pixel_details.html",
"post.json",
"profile-method-info",
"project-id",
"publisha.1.json",
"publish_and_moderate",
"Ronna McDaniel",
"rtb-h",
"runs",
"sign-up",
"some-uuid-maybe",
"stable-4.0-version.json",
"StartUpCheckout",
"Steve Flunk",
"sync_a9",
"Ted Cruz",
"test.png",
"token",
"ToList",
"v2.1.3",
"VersionCheck.php",
"v Rusiji",
"Walnut St",
"web_widget",
"zoom_in.cur",
"xray",
"web",
"vipbets1",
"trcc",
"fbpixel",
// TODO below
// "tcfv2",
// "Matt-cartoon-255x206px-small.png",
// "TheTelegraph_portal_white-320-small.png",
// "testdata-10kB.js",
}
for _, str := range cases {
if IsGibberish(str) {
t.Errorf("Mistakenly true: %s", str)
}
}
}
func TestPositive(t *testing.T) {
cases := []string{
"0a0d0174-b338-4520-a1c3-24f7e3d5ec50.html",
"1024807212418223",
"11ca096cbc224a67360493d44a9903",
"1553183382779",
"1554507871",
"19180481",
"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
"456795af-b48f-4a8d-9b37-3e932622c2f0",
"601a2bdcc5b69137248ddbbf",
"60fe9aaeaefe2400012df94f",
"610bc3fd5a77a7fa25033fb0",
"610bd0315a77a7fa25034368",
"610bd0315a77a7fa25034368zh",
"6120c057c7a97b03f6986f1b",
"710a462e",
"730970532670-compute@developer.gserviceaccount.com",
"819db2242a648b305395537022523d65",
"952bea17-3776-11ea-9341-42010a84012a",
"a3226860758.html",
"AAAA028295945",
"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4",
"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
"bnjksfd897345nl098asd53412kl98",
"c738338322370b47a79251f7510dd", // prefixed hex
"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
"css/login.0f48c49a34eb53ea4623.min.css",
"d_fLLxlhzDilixeBEimaZ5",
"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
"e8782afc112720300c049ff124434b79",
"fb6cjraf9cejut2a",
"i-0236530c66ed02200",
"JEHJW4BKVFDRTMTUQLHKK5WVAU",
"john.dow.1981@protonmail.com",
"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
"MNUTGVFMGLEMFTBH0XSE5E02F6J2DS",
"n63nd45qsj",
"n9z9QGNiz",
"NC4WTmcy",
"proxy.3d2100fd7107262ecb55ce6847f01fa5.html",
"QgAAAC6zw0qH2DJtnXe8Z7rUJP0FgAFKkOhcHdFWzL1ZYggtwBgiB3LSoele9o3ZqFh7iCBhHbVLAnMuJ0HF8hEw7UKecE6wd-MBXgeRMdubGydhAMZSmuUjRpqplML40bmrb8VjJKNZswD1Cg",
"QgAAAC6zw0qH2DJtnXe8Z7rUJP0rG4sjLa_KVLlww5WEDJ__30J15en-K_6Y68jb_rU93e2TFY6fb0MYiQ1UrLNMQufqODHZUl39Lo6cXAOVOThjAMZSmuVH7n85JOYSCgzpvowMAVueGG0Xxg",
"qwerqwerasdfqwer@protonmai.com",
"r-ext-5579e00a95c90",
"r-ext-5579e8b12f11e",
"r-v4-5c92513c9e04a",
"r-v4-5c92513c9e04a-0-c.monitoring.json",
"segments-1563566437171.639994",
"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
"sp_dxJTfx11_576742227280287872",
"sp_NnUPB5wj_601a2bdcc5b69137248ddbbf",
"sp_NxITuoE4_premiumchron-article-14302157_c_ryGQBs_r_yIWvwP",
"t_52d94268-8810-4a7e-ba87-ffd657a6752f",
"timeouts-1563566437171.639994",
"u_YPF3GsGKMo02",
"0000000000 65535 f",
"0000000178 00000 n",
"0-10000",
"01526123,",
"0,18168,183955,3,4,1151616,5663,731,223,5104,207,3204,10,1051,175,364,1435,4,60,576,241,383,246,5,1102",
"05/10/2020",
"14336456724940333",
"fb6cjraf9cejut2a",
"JEHJW4BKVFDRTMTUQLHKK5WVAU",
// TODO
/*
"0,20",
"0.001",
"YISAtiX1",
"Fxvd1timk", // questionable
"B4GCSkORAJs",
"D_4EDAqenHQ",
"EICJp29EGOk",
"Fxvd1timk",
"GTqMZELYfQQ",
"GZPTpLPEGmwHGWPC",
"_HChnE9NDPY",
"NwhjgIWHgGg",
"production/tsbqksph4xswqjexfbec",
"p/u/bguhrxupr23mw3nwxcrw",
"nRSNapbJZnc",
"zgfpbtolciznub5egzxk",
"zufnu7aimadua9wrgwwo",
"zznto1jzch9yjsbtbrul",
*/
}
for _, str := range cases {
if !IsGibberish(str) {
t.Errorf("Mistakenly false: %s", str)
}
}
}
func TestVersionStrings(t *testing.T) {
cases := []string{
"1.0",
"1.0.0",
"v2.1.3",
"2.1.73",
}
for _, str := range cases {
if !IsVersionString(str) {
t.Errorf("Mistakenly false: %s", str)
}
}
}

77
agent/pkg/oas/ignores.go Normal file
View File

@ -0,0 +1,77 @@
package oas
import "strings"
var ignoredExtensions = []string{"gif", "svg", "css", "png", "ico", "js", "woff2", "woff", "jpg", "jpeg", "swf", "ttf", "map", "webp", "otf", "mp3"}
var ignoredCtypePrefixes = []string{"image/", "font/", "video/", "audio/", "text/javascript"}
var ignoredCtypes = []string{"application/javascript", "application/x-javascript", "text/css", "application/font-woff2", "application/font-woff", "application/x-font-woff"}
var ignoredHeaders = []string{
"a-im", "accept",
"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-range", "content-type", "cookie",
"date", "dnt", "expect", "forwarded", "from", "front-end-https", "host", "http2-settings",
"max-forwards", "origin", "pragma", "proxy-authorization", "proxy-connection", "range", "referer",
"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests", "x-download-options",
"server", "user-agent", "via", "warning", "strict-transport-security", "x-permitted-cross-domain-policies",
"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data", "x-dns-prefetch-control",
"x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh",
"x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection",
"x-wap-profile", "x-scheme", "status", "x-cache", "x-application-context", "retry-after",
"newrelic", "x-cloud-trace-context", "sentry-trace", "x-cache-hits", "x-served-by", "x-span-name",
"expires", "set-cookie", "p3p", "content-security-policy", "content-security-policy-report-only",
"last-modified", "content-language", "x-varnish", "true-client-ip", "akamai-origin-hop",
"keep-alive", "etag", "alt-svc", "x-csrf-token", "x-ua-compatible", "vary", "x-powered-by",
"age", "allow", "www-authenticate", "expect-ct", "timing-allow-origin", "referrer-policy",
"x-aspnet-version", "x-aspnetmvc-version", "x-timer", "x-abuse-info", "x-mod-pagespeed",
"duration_ms", // UP9 custom
}
var ignoredHeaderPrefixes = []string{
":", "accept-", "access-control-", "if-", "sec-", "grpc-",
"x-forwarded-", "x-original-", "cf-",
"x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-",
"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-", "x-goog-",
}
func isCtypeIgnored(ctype string) bool {
for _, prefix := range ignoredCtypePrefixes {
if strings.HasPrefix(ctype, prefix) {
return true
}
}
for _, toIgnore := range ignoredCtypes {
if ctype == toIgnore {
return true
}
}
return false
}
func isExtIgnored(path string) bool {
for _, extIgn := range ignoredExtensions {
if strings.HasSuffix(path, "."+extIgn) {
return true
}
}
return false
}
func isHeaderIgnored(name string) bool {
name = strings.ToLower(name)
for _, ignore := range ignoredHeaders {
if name == ignore {
return true
}
}
for _, prefix := range ignoredHeaderPrefixes {
if strings.HasPrefix(name, prefix) {
return true
}
}
return false
}

View File

@ -0,0 +1,96 @@
{
"openapi": "3.1.0",
"info": {
"title": "http://carts",
"description": "Mizu observed 3 entries (0 failed), at 2.287 hits/s, average response time is 0.017 seconds",
"version": "1.0"
},
"servers": [
{
"url": "http://carts"
}
],
"paths": {
"/carts/{cartId}/items": {
"get": {
"summary": "/carts/{cartId}/items",
"description": "Mizu observed 3 entries (0 failed), at 2.287 hits/s, average response time is 0.017 seconds",
"operationId": "84c9b926-1f73-4ab4-b381-3c124528959f",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": [
{
"id": "60fe98fb86c0fc000869a90c",
"itemId": "3395a43e-2d88-40de-b95f-e00e1502085b",
"quantity": 1,
"unitPrice": 18
}
],
"x-sample-entry": "000000000000000000000010"
}
},
"x-sample-entry": "000000000000000000000010"
}
},
"x-counters-per-source": {
"some-source": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298058.3798368,
"lastSeen": 1627298065.2397773,
"sumRT": 0.05,
"sumDuration": 6.859940528869629
}
},
"x-counters-total": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298058.3798368,
"lastSeen": 1627298065.2397773,
"sumRT": 0.05,
"sumDuration": 6.859940528869629
},
"x-last-seen-ts": 1627298065.2397773,
"x-sample-entry": "000000000000000000000010"
},
"parameters": [
{
"name": "cartId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "mHK0P7zTktmV1zv57iWAvCTd43FFMHap"
}
},
"x-sample-entry": "000000000000000000000010"
}
]
}
},
"x-counters-per-source": {
"some-source": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298058.3798368,
"lastSeen": 1627298065.2397773,
"sumRT": 0.05,
"sumDuration": 6.859940528869629
}
},
"x-counters-total": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298058.3798368,
"lastSeen": 1627298065.2397773,
"sumRT": 0.05,
"sumDuration": 6.859940528869629
}
}

View File

@ -0,0 +1,485 @@
{
"openapi": "3.1.0",
"info": {
"title": "Preloaded",
"description": "Test file for loading pre-existing OAS",
"version": "0.1"
},
"paths": {
"/catalogue": {
"get": {
"tags": [
"catalogue"
],
"summary": "/catalogue",
"description": "Mizu observed 3 entries (0 failed), at 2.647 hits/s, average response time is 0.008 seconds",
"operationId": "dd6c3dbe-6b6b-4ddd-baed-757e237ddb8a",
"parameters": [
{
"name": "page",
"in": "query",
"required": false,
"style": "form",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "1"
}
},
"x-sample-entry": "000000000000000000000002"
},
{
"name": "size",
"in": "query",
"required": true,
"style": "form",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "6"
},
"example #1": {
"value": "3"
},
"example #2": {
"value": "5"
}
},
"x-sample-entry": "000000000000000000000011"
},
{
"name": "tags",
"in": "query",
"required": false,
"style": "form",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": ""
},
"example #1": {
"value": "blue"
}
},
"x-sample-entry": "000000000000000000000007"
},
{
"name": "sort",
"in": "query",
"required": false,
"style": "form",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "id"
}
},
"x-sample-entry": "000000000000000000000007"
}
],
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": [
{
"count": 1,
"description": "Socks fit for a Messiah. You too can experience walking in water with these special edition beauties. Each hole is lovingly proggled to leave smooth edges. The only sock approved by a higher power.",
"id": "03fef6ac-1896-4ce8-bd69-b798f85c6e0b",
"imageUrl": [
"/catalogue/images/holy_1.jpeg",
"/catalogue/images/holy_2.jpeg"
],
"name": "Holy",
"price": 99.99,
"tag": [
"action",
"magic"
]
},
{
"count": 438,
"description": "proident occaecat irure et excepteur labore minim nisi amet irure",
"id": "3395a43e-2d88-40de-b95f-e00e1502085b",
"imageUrl": [
"/catalogue/images/colourful_socks.jpg",
"/catalogue/images/colourful_socks.jpg"
],
"name": "Colourful",
"price": 18,
"tag": [
"brown",
"blue"
]
},
{
"count": 820,
"description": "Ready for action. Engineers: be ready to smash that next bug! Be ready, with these super-action-sport-masterpieces. This particular engineer was chased away from the office with a stick.",
"id": "510a0d7e-8e83-4193-b483-e27e09ddc34d",
"imageUrl": [
"/catalogue/images/puma_1.jpeg",
"/catalogue/images/puma_2.jpeg"
],
"name": "SuperSport XL",
"price": 15,
"tag": [
"sport",
"formal",
"black"
]
},
{
"count": 738,
"description": "A mature sock, crossed, with an air of nonchalance.",
"id": "808a2de1-1aaa-4c25-a9b9-6612e8f29a38",
"imageUrl": [
"/catalogue/images/cross_1.jpeg",
"/catalogue/images/cross_2.jpeg"
],
"name": "Crossed",
"price": 17.32,
"tag": [
"blue",
"action",
"red",
"formal"
]
},
{
"count": 808,
"description": "enim officia aliqua excepteur esse deserunt quis aliquip nostrud anim",
"id": "819e1fbf-8b7e-4f6d-811f-693534916a8b",
"imageUrl": [
"/catalogue/images/WAT.jpg",
"/catalogue/images/WAT2.jpg"
],
"name": "Figueroa",
"price": 14,
"tag": [
"green",
"formal",
"blue"
]
},
{
"count": 175,
"description": "consequat amet cupidatat minim laborum tempor elit ex consequat in",
"id": "837ab141-399e-4c1f-9abc-bace40296bac",
"imageUrl": [
"/catalogue/images/catsocks.jpg",
"/catalogue/images/catsocks2.jpg"
],
"name": "Cat socks",
"price": 15,
"tag": [
"brown",
"formal",
"green"
]
}
],
"x-sample-entry": "000000000000000000000011"
}
},
"x-sample-entry": "000000000000000000000011"
}
},
"x-counters-per-source": {
"some-source": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298057.7849188,
"lastSeen": 1627298065.7258668,
"sumRT": 0.024999999999999998,
"sumDuration": 7.940948009490967
}
},
"x-counters-total": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298057.7849188,
"lastSeen": 1627298065.7258668,
"sumRT": 0.024999999999999998,
"sumDuration": 7.940948009490967
},
"x-last-seen-ts": 1627298065.7258668,
"x-sample-entry": "000000000000000000000011"
}
},
"/catalogue/size": {
"get": {
"tags": [
"catalogue"
],
"summary": "/catalogue/size",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.013 seconds",
"operationId": "2315e69d-9d66-48cf-b3d3-fec9c30bd28b",
"parameters": [
{
"name": "tags",
"in": "query",
"required": true,
"style": "form",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": ""
}
},
"x-sample-entry": "000000000000000000000001"
},
{
"name": "x-some",
"in": "header",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "demo val"
}
},
"x-sample-entry": "000000000000000000000001"
}
],
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": {
"err": null,
"size": 9
},
"x-sample-entry": "000000000000000000000001"
}
},
"x-sample-entry": "000000000000000000000001"
}
},
"x-counters-per-source": {
"some-source": {
"entries": 1,
"failures": 0,
"firstSeen": 1627298057.7841518,
"lastSeen": 1627298057.7841518,
"sumRT": 0.013,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1627298057.7841518,
"lastSeen": 1627298057.7841518,
"sumRT": 0.013,
"sumDuration": 0
},
"x-last-seen-ts": 1627298057.7841518,
"x-sample-entry": "000000000000000000000001"
}
},
"/catalogue/{id}": {
"get": {
"tags": [
"catalogue"
],
"summary": "/catalogue/{id}",
"description": "Mizu observed 4 entries (0 failed), at 1.899 hits/s, average response time is 0.003 seconds",
"parameters": [
{
"name": "non-required-header",
"in": "header",
"required": false,
"style": "simple",
"schema": {
"type": "string"
},
"example": "some-uuid-maybe"
},
{
"name": "x-some",
"in": "header",
"required": false,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "demoval"
}
},
"x-sample-entry": "000000000000000000000004"
}
],
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": {
"count": 438,
"description": "proident occaecat irure et excepteur labore minim nisi amet irure",
"id": "3395a43e-2d88-40de-b95f-e00e1502085b",
"imageUrl": [
"/catalogue/images/colourful_socks.jpg",
"/catalogue/images/colourful_socks.jpg"
],
"name": "Colourful",
"price": 18,
"tag": [
"brown",
"blue"
]
},
"x-sample-entry": "000000000000000000000012"
}
},
"x-sample-entry": "000000000000000000000012"
}
},
"x-counters-per-source": {
"some-source": {
"entries": 4,
"failures": 0,
"firstSeen": 1627298058.1315014,
"lastSeen": 1627298065.7293031,
"sumRT": 0.013999999999999999,
"sumDuration": 7.597801685333252
}
},
"x-counters-total": {
"entries": 4,
"failures": 0,
"firstSeen": 1627298058.1315014,
"lastSeen": 1627298065.7293031,
"sumRT": 0.013999999999999999,
"sumDuration": 7.597801685333252
},
"x-last-seen-ts": 1627298065.7293031,
"x-sample-entry": "000000000000000000000012"
},
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "3395a43e-2d88-40de-b95f-e00e1502085b"
},
"example #1": {
"value": "808a2de1-1aaa-4c25-a9b9-6612e8f29a38"
}
},
"example": "some-uuid-maybe",
"x-sample-entry": "000000000000000000000012"
}
]
},
"/catalogue/{id}/details": {
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"example": "some-uuid-maybe"
}
]
},
"/tags": {
"get": {
"summary": "/tags",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.007 seconds",
"operationId": "c4d7d0ed-1a78-4370-a049-efe3abc631a6",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": {
"err": null,
"tags": [
"brown",
"geek",
"formal",
"blue",
"skin",
"red",
"action",
"sport",
"black",
"magic",
"green"
]
},
"x-sample-entry": "000000000000000000000003"
}
},
"x-sample-entry": "000000000000000000000003"
}
},
"x-counters-per-source": {
"some-source": {
"entries": 1,
"failures": 0,
"firstSeen": 1627298057.7841816,
"lastSeen": 1627298057.7841816,
"sumRT": 0.007,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1627298057.7841816,
"lastSeen": 1627298057.7841816,
"sumRT": 0.007,
"sumDuration": 0
},
"x-last-seen-ts": 1627298057.7841816,
"x-sample-entry": "000000000000000000000003"
}
}
},
"x-counters-per-source": {
"some-source": {
"entries": 9,
"failures": 0,
"firstSeen": 1627298057.7841518,
"lastSeen": 1627298065.7293031,
"sumRT": 0.05899999999999999,
"sumDuration": 15.538749694824219
}
},
"x-counters-total": {
"entries": 9,
"failures": 0,
"firstSeen": 1627298057.7841518,
"lastSeen": 1627298065.7293031,
"sumRT": 0.05899999999999999,
"sumDuration": 15.538749694824219
}
}

View File

@ -0,0 +1,897 @@
{
"openapi": "3.1.0",
"info": {
"title": "https://httpbin.org",
"description": "Mizu observed 19 entries (0 failed), at 0.106 hits/s, average response time is 0.172 seconds",
"version": "1.0"
},
"servers": [
{
"url": "https://httpbin.org"
}
],
"paths": {
"/appears-once": {
"get": {
"summary": "/appears-once",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
"operationId": "2d34623e-fde8-4720-8390-9a7439051755",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000004"
}
},
"x-sample-entry": "000000000000000000000004"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750580.0471218,
"lastSeen": 1567750580.0471218,
"sumRT": 0.63,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750580.0471218,
"lastSeen": 1567750580.0471218,
"sumRT": 0.63,
"sumDuration": 0
},
"x-last-seen-ts": 1567750580.0471218,
"x-sample-entry": "000000000000000000000004"
}
},
"/appears-twice": {
"get": {
"summary": "/appears-twice",
"description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds",
"operationId": "9c5330f3-8062-468b-b5a3-df1ad82b4846",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000006"
}
},
"x-sample-entry": "000000000000000000000006"
}
},
"x-counters-per-source": {
"": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.7471218,
"lastSeen": 1567750581.7471218,
"sumRT": 1.26,
"sumDuration": 1
}
},
"x-counters-total": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.7471218,
"lastSeen": 1567750581.7471218,
"sumRT": 1.26,
"sumDuration": 1
},
"x-last-seen-ts": 1567750581.7471218,
"x-sample-entry": "000000000000000000000006"
}
},
"/body-optional": {
"post": {
"summary": "/body-optional",
"description": "Mizu observed 3 entries (0 failed), at 0.003 hits/s, average response time is 0.001 seconds",
"operationId": "34f3d66c-b1f7-4dca-9cab-987fcc8ae472",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000012"
}
},
"x-sample-entry": "000000000000000000000012"
}
},
"x-counters-per-source": {
"": {
"entries": 3,
"failures": 0,
"firstSeen": 1567750581.7471218,
"lastSeen": 1567750581.757122,
"sumRT": 0.003,
"sumDuration": 0.010000228881835938
}
},
"x-counters-total": {
"entries": 3,
"failures": 0,
"firstSeen": 1567750581.7471218,
"lastSeen": 1567750581.757122,
"sumRT": 0.003,
"sumDuration": 0.010000228881835938
},
"x-last-seen-ts": 1567750581.757122,
"x-sample-entry": "000000000000000000000012",
"requestBody": {
"description": "Generic request body",
"content": {
"application/json": {
"example": "{\"key\", \"val\"}",
"x-sample-entry": "000000000000000000000011"
}
},
"x-sample-entry": "000000000000000000000012"
}
}
},
"/body-required": {
"post": {
"summary": "/body-required",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "ff6add53-ab1c-4d4e-b590-0835fa318276",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000013"
}
},
"x-sample-entry": "000000000000000000000013"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750581.757122,
"lastSeen": 1567750581.757122,
"sumRT": 0.001,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750581.757122,
"lastSeen": 1567750581.757122,
"sumRT": 0.001,
"sumDuration": 0
},
"x-last-seen-ts": 1567750581.757122,
"x-sample-entry": "000000000000000000000013",
"requestBody": {
"description": "Generic request body",
"content": {
"": {
"example": "body exists",
"x-sample-entry": "000000000000000000000013"
}
},
"required": true,
"x-sample-entry": "000000000000000000000013"
}
}
},
"/form-multipart": {
"post": {
"summary": "/form-multipart",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "153f0925-9fc7-4e9f-9d33-f1470f25f0f7",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"example": {},
"x-sample-entry": "000000000000000000000009"
}
},
"x-sample-entry": "000000000000000000000009"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.7471218,
"lastSeen": 1567750582.7471218,
"sumRT": 0.001,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.7471218,
"lastSeen": 1567750582.7471218,
"sumRT": 0.001,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.7471218,
"x-sample-entry": "000000000000000000000009",
"requestBody": {
"description": "Generic request body",
"content": {
"multipart/form-data": {
"schema": {
"type": "object",
"required": [
"file",
"path"
],
"properties": {
"file": {
"type": "string",
"contentMediaType": "application/json",
"examples": [
"{\"functions\": 123}"
]
},
"path": {
"type": "string",
"examples": [
"/content/components"
]
}
}
},
"example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n",
"x-sample-entry": "000000000000000000000009"
}
},
"required": true,
"x-sample-entry": "000000000000000000000009"
}
}
},
"/form-urlencoded": {
"post": {
"summary": "/form-urlencoded",
"description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds",
"operationId": "c92189f5-5636-46eb-ac71-92b17941a568",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000008"
}
},
"x-sample-entry": "000000000000000000000008"
}
},
"x-counters-per-source": {
"": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.7471218,
"lastSeen": 1567750581.7471218,
"sumRT": 0.002,
"sumDuration": 1
}
},
"x-counters-total": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.7471218,
"lastSeen": 1567750581.7471218,
"sumRT": 0.002,
"sumDuration": 1
},
"x-last-seen-ts": 1567750581.7471218,
"x-sample-entry": "000000000000000000000008",
"requestBody": {
"description": "Generic request body",
"content": {
"application/x-www-form-urlencoded": {
"schema": {
"type": "object",
"required": [
"agent-id",
"callback-url",
"token"
],
"properties": {
"agent-id": {
"type": "string",
"examples": [
"ade"
]
},
"callback-url": {
"type": "string",
"examples": [
""
]
},
"optional": {
"type": "string",
"examples": [
"another"
]
},
"token": {
"type": "string",
"examples": [
"sometoken",
"sometoken-second-val"
]
}
}
},
"example": "agent-id=ade\u0026callback-url=\u0026token=sometoken",
"x-sample-entry": "000000000000000000000008"
}
},
"required": true,
"x-sample-entry": "000000000000000000000008"
}
}
},
"/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "85270437-7aae-4a5b-b988-3662092463d0",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000014"
}
},
"x-sample-entry": "000000000000000000000014"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582,
"lastSeen": 1567750582,
"sumRT": 0.001,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582,
"lastSeen": 1567750582,
"sumRT": 0.001,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582,
"x-sample-entry": "000000000000000000000014"
},
"parameters": [
{
"name": "prefixgibberishfineId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "234324"
}
},
"x-sample-entry": "000000000000000000000014"
}
]
},
"/param-patterns/{parampatternId}": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}",
"description": "Mizu observed 2 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "da597734-1cf5-4d3b-917b-6b02dacf7b7b",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000018"
}
},
"x-sample-entry": "000000000000000000000018"
}
},
"x-counters-per-source": {
"": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750582.000003,
"lastSeen": 1567750582.000004,
"sumRT": 0.002,
"sumDuration": 9.5367431640625e-7
}
},
"x-counters-total": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750582.000003,
"lastSeen": 1567750582.000004,
"sumRT": 0.002,
"sumDuration": 9.5367431640625e-7
},
"x-last-seen-ts": 1567750582.000004,
"x-sample-entry": "000000000000000000000018"
},
"parameters": [
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/param-patterns/{parampatternId}/1": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}/1",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "e965a245-9cfc-48ed-94e1-f765eadb3960",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000015"
}
},
"x-sample-entry": "000000000000000000000015"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.000001,
"lastSeen": 1567750582.000001,
"sumRT": 0.001,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.000001,
"lastSeen": 1567750582.000001,
"sumRT": 0.001,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.000001,
"x-sample-entry": "000000000000000000000015"
},
"parameters": [
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/param-patterns/{parampatternId}/static": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}/static",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "7af420dc-f8b7-450f-8f6f-18b039aa3cde",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000016"
}
},
"x-sample-entry": "000000000000000000000016"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.000002,
"lastSeen": 1567750582.000002,
"sumRT": 0.001,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.000002,
"lastSeen": 1567750582.000002,
"sumRT": 0.001,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.000002,
"x-sample-entry": "000000000000000000000016"
},
"parameters": [
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/param-patterns/{parampatternId}/{param1}": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}/{param1}",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "02a1771d-2d50-4a8c-8be2-29c7e59b8435",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000019"
}
},
"x-sample-entry": "000000000000000000000019"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.000002,
"lastSeen": 1567750582.000002,
"sumRT": 0.001,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.000002,
"lastSeen": 1567750582.000002,
"sumRT": 0.001,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.000002,
"x-sample-entry": "000000000000000000000019"
},
"parameters": [
{
"name": "param1",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "23421"
}
},
"x-sample-entry": "000000000000000000000019"
},
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/{Id}": {
"get": {
"summary": "/{Id}",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
"operationId": "77ec4910-d47a-46a5-8234-fb80a11034b4",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000003"
}
},
"x-sample-entry": "000000000000000000000003"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750579.7471218,
"lastSeen": 1567750579.7471218,
"sumRT": 0.63,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750579.7471218,
"lastSeen": 1567750579.7471218,
"sumRT": 0.63,
"sumDuration": 0
},
"x-last-seen-ts": 1567750579.7471218,
"x-sample-entry": "000000000000000000000003"
},
"parameters": [
{
"name": "Id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
},
"example #1": {
"value": "952bea17-3776-11ea-9341-42010a84012a"
}
},
"x-sample-entry": "000000000000000000000003"
}
]
},
"/{Id}/sub1": {
"get": {
"summary": "/{Id}/sub1",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds",
"operationId": "198675eb-9faf-407b-83fa-0483a730bbbe",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"text/html": {
"x-sample-entry": "000000000000000000000001"
}
},
"x-sample-entry": "000000000000000000000001"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750483.864529,
"lastSeen": 1567750483.864529,
"sumRT": 0.111,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750483.864529,
"lastSeen": 1567750483.864529,
"sumRT": 0.111,
"sumDuration": 0
},
"x-last-seen-ts": 1567750483.864529,
"x-sample-entry": "000000000000000000000001"
},
"parameters": [
{
"name": "Id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
},
"example #1": {
"value": "952bea17-3776-11ea-9341-42010a84012a"
}
},
"x-sample-entry": "000000000000000000000003"
}
]
},
"/{Id}/sub2": {
"get": {
"summary": "/{Id}/sub2",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
"operationId": "31d880f1-152f-4dd6-84a7-463e13b694a5",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000002"
}
},
"x-sample-entry": "000000000000000000000002"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750578.7471218,
"lastSeen": 1567750578.7471218,
"sumRT": 0.63,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750578.7471218,
"lastSeen": 1567750578.7471218,
"sumRT": 0.63,
"sumDuration": 0
},
"x-last-seen-ts": 1567750578.7471218,
"x-sample-entry": "000000000000000000000002"
},
"parameters": [
{
"name": "Id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
},
"example #1": {
"value": "952bea17-3776-11ea-9341-42010a84012a"
}
},
"x-sample-entry": "000000000000000000000003"
}
]
}
},
"x-counters-per-source": {
"": {
"entries": 19,
"failures": 0,
"firstSeen": 1567750483.864529,
"lastSeen": 1567750582.7471218,
"sumRT": 3.273999999999999,
"sumDuration": 2.0100011825561523
}
},
"x-counters-total": {
"entries": 19,
"failures": 0,
"firstSeen": 1567750483.864529,
"lastSeen": 1567750582.7471218,
"sumRT": 3.273999999999999,
"sumDuration": 2.0100011825561523
}
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,142 @@
package oas
import (
"encoding/json"
"net/url"
"sync"
"github.com/up9inc/mizu/agent/pkg/har"
"github.com/up9inc/mizu/logger"
"github.com/up9inc/mizu/tap/api"
)
var (
syncOnce sync.Once
instance *defaultOasGenerator
)
type OasGeneratorSink interface {
HandleEntry(mizuEntry *api.Entry)
}
type OasGenerator interface {
Start()
Stop()
IsStarted() bool
GetServiceSpecs() *sync.Map
}
type defaultOasGenerator struct {
started bool
serviceSpecs *sync.Map
maxExampleLen int
}
func GetDefaultOasGeneratorInstance(maxExampleLen int) *defaultOasGenerator {
syncOnce.Do(func() {
instance = NewDefaultOasGenerator(maxExampleLen)
logger.Log.Debug("OAS Generator Initialized")
})
return instance
}
func (g *defaultOasGenerator) Start() {
g.started = true
}
func (g *defaultOasGenerator) Stop() {
if !g.started {
return
}
g.started = false
g.reset()
}
func (g *defaultOasGenerator) IsStarted() bool {
return g.started
}
func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry) {
if !g.started {
return
}
if mizuEntry.Protocol.Name == "http" {
dest := mizuEntry.Destination.Name
if dest == "" {
logger.Log.Debugf("OAS: Unresolved entry %d", mizuEntry.Id)
return
}
entry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
if err != nil {
logger.Log.Warningf("Failed to turn MizuEntry %d into HAR Entry: %s", mizuEntry.Id, err)
return
}
entryWSource := &EntryWithSource{
Entry: *entry,
Source: mizuEntry.Source.Name,
Destination: dest,
Id: mizuEntry.Id,
}
g.handleHARWithSource(entryWSource)
} else {
logger.Log.Debugf("OAS: Unsupported protocol in entry %d: %s", mizuEntry.Id, mizuEntry.Protocol.Name)
}
}
func (g *defaultOasGenerator) handleHARWithSource(entryWSource *EntryWithSource) {
entry := entryWSource.Entry
gen := g.getGen(entryWSource.Destination, entry.Request.URL)
opId, err := gen.feedEntry(entryWSource)
if err != nil {
txt, suberr := json.Marshal(entry)
if suberr == nil {
logger.Log.Debugf("Problematic entry: %s", txt)
}
logger.Log.Warningf("Failed processing entry %d: %s", entryWSource.Id, err)
return
}
logger.Log.Debugf("Handled entry %s as opId: %s", entryWSource.Id, opId) // TODO: set opId back to entry?
}
func (g *defaultOasGenerator) getGen(dest string, urlStr string) *SpecGen {
u, err := url.Parse(urlStr)
if err != nil {
logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", urlStr, err)
}
val, found := g.serviceSpecs.Load(dest)
var gen *SpecGen
if !found {
gen = NewGen(u.Scheme + "://" + dest)
gen.MaxExampleLen = g.maxExampleLen
g.serviceSpecs.Store(dest, gen)
} else {
gen = val.(*SpecGen)
}
return gen
}
func (g *defaultOasGenerator) reset() {
g.serviceSpecs = &sync.Map{}
}
func (g *defaultOasGenerator) GetServiceSpecs() *sync.Map {
return g.serviceSpecs
}
func NewDefaultOasGenerator(maxExampleLen int) *defaultOasGenerator {
return &defaultOasGenerator{
started: false,
serviceSpecs: &sync.Map{},
maxExampleLen: maxExampleLen,
}
}

View File

@ -0,0 +1,45 @@
package oas
import (
"encoding/json"
"github.com/up9inc/mizu/agent/pkg/har"
"testing"
"time"
)
func TestOASGen(t *testing.T) {
gen := GetDefaultOasGeneratorInstance(-1)
e := new(har.Entry)
err := json.Unmarshal([]byte(`{"startedDateTime": "20000101","request": {"url": "https://host/path", "method": "GET"}, "response": {"status": 200}}`), e)
if err != nil {
panic(err)
}
ews := &EntryWithSource{
Destination: "some",
Entry: *e,
}
gen.Start()
gen.handleHARWithSource(ews)
g, ok := gen.serviceSpecs.Load("some")
if !ok {
panic("Failed")
}
sg := g.(*SpecGen)
spec, err := sg.GetSpec()
if err != nil {
panic(err)
}
specText, _ := json.Marshal(spec)
t.Log(string(specText))
if !gen.IsStarted() {
t.Errorf("Should be started")
}
time.Sleep(100 * time.Millisecond)
gen.Stop()
}

748
agent/pkg/oas/specgen.go Normal file
View File

@ -0,0 +1,748 @@
package oas
import (
"encoding/json"
"errors"
"io"
"io/ioutil"
"mime"
"mime/multipart"
"net/textproto"
"net/url"
"sort"
"strconv"
"strings"
"sync"
"github.com/chanced/openapi"
"github.com/google/uuid"
"github.com/nav-inc/datetime"
"github.com/up9inc/mizu/logger"
"github.com/up9inc/mizu/agent/pkg/har"
"time"
)
const LastSeenTS = "x-last-seen-ts"
const CountersTotal = "x-counters-total"
const CountersPerSource = "x-counters-per-source"
const SampleId = "x-sample-entry"
type EntryWithSource struct {
Source string
Destination string
Entry har.Entry
Id string
}
type reqResp struct { // hello, generics in Go
Req *har.Request
Resp *har.Response
}
type SpecGen struct {
MaxExampleLen int // -1 unlimited, 0 and above sets limit
oas *openapi.OpenAPI
tree *Node
lock sync.Mutex
}
func NewGen(server string) *SpecGen {
spec := new(openapi.OpenAPI)
spec.Version = "3.1.0"
info := openapi.Info{Title: server}
info.Version = "1.0"
spec.Info = &info
spec.Paths = &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
spec.Servers = make([]*openapi.Server, 0)
spec.Servers = append(spec.Servers, &openapi.Server{URL: server})
gen := SpecGen{
oas: spec,
tree: new(Node),
MaxExampleLen: -1,
}
return &gen
}
func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
g.oas = oas
g.tree = new(Node)
for pathStr, pathObj := range oas.Paths.Items {
pathSplit := strings.Split(string(pathStr), "/")
g.tree.getOrSet(pathSplit, pathObj, "")
// clean "last entry timestamp" markers from the past
for _, pathAndOp := range g.tree.listOps() {
delete(pathAndOp.op.Extensions, LastSeenTS)
}
}
}
func (g *SpecGen) feedEntry(entryWithSource *EntryWithSource) (string, error) {
g.lock.Lock()
defer g.lock.Unlock()
opId, err := g.handlePathObj(entryWithSource)
if err != nil {
return "", err
}
// NOTE: opId can be empty for some failed entries
return opId, err
}
func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
g.lock.Lock()
defer g.lock.Unlock()
g.tree.compact()
counters := CounterMaps{counterTotal: Counter{}, counterMapTotal: CounterMap{}}
for _, pathAndOp := range g.tree.listOps() {
opObj := pathAndOp.op
if opObj.Summary == "" {
opObj.Summary = pathAndOp.path
}
err := counters.processOp(opObj)
if err != nil {
return nil, err
}
}
err := counters.processOas(g.oas)
if err != nil {
return nil, err
}
// put paths back from tree into OAS
g.oas.Paths = g.tree.listPaths()
suggestTags(g.oas)
g.oas.Info.Description = setCounterMsgIfOk(g.oas.Info.Description, &counters.counterTotal)
// to make a deep copy, no better idea than marshal+unmarshal
specText, err := json.MarshalIndent(g.oas, "", "\t")
if err != nil {
return nil, err
}
spec := new(openapi.OpenAPI)
err = json.Unmarshal(specText, spec)
if err != nil {
return nil, err
}
return spec, err
}
func suggestTags(oas *openapi.OpenAPI) {
paths := getPathsKeys(oas.Paths.Items)
sort.Strings(paths) // make it stable in case of multiple candidates
for len(paths) > 0 {
group := make([]string, 0)
group = append(group, paths[0])
paths = paths[1:]
pathsClone := append(paths[:0:0], paths...)
for _, path := range pathsClone {
if getSimilarPrefix([]string{group[0], path}) != "" {
group = append(group, path)
paths = deleteFromSlice(paths, path)
}
}
common := getSimilarPrefix(group)
if len(group) > 1 {
for _, path := range group {
pathObj := oas.Paths.Items[openapi.PathValue(path)]
for _, op := range getOps(pathObj) {
if op.Tags == nil {
op.Tags = make([]string, 0)
}
// only add tags if not present
if len(op.Tags) == 0 {
op.Tags = append(op.Tags, common)
}
}
}
}
}
}
func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
keys := make([]string, len(mymap))
i := 0
for k := range mymap {
keys[i] = string(k)
i++
}
return keys
}
func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error) {
entry := entryWithSource.Entry
urlParsed, err := url.Parse(entry.Request.URL)
if err != nil {
return "", err
}
if isExtIgnored(urlParsed.Path) {
logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path)
return "", nil
}
if entry.Request.Method == "OPTIONS" {
logger.Log.Debugf("Dropped traffic entry due to its method: %s %s", entry.Request.Method, urlParsed.Path)
return "", nil
}
ctype := getRespCtype(&entry.Response)
if isCtypeIgnored(ctype) {
logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype)
return "", nil
}
if entry.Response.Status < 100 {
logger.Log.Debugf("Dropped traffic entry due to status<100: %s", entry.StartedDateTime)
return "", nil
}
if entry.Response.Status == 301 || entry.Response.Status == 308 {
logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
return "", nil
}
if entry.Response.Status == 502 || entry.Response.Status == 503 || entry.Response.Status == 504 {
logger.Log.Debugf("Dropped traffic entry due to temporary server error: %s", entry.StartedDateTime)
return "", nil
}
var split []string
if urlParsed.RawPath != "" {
split = strings.Split(urlParsed.RawPath, "/")
} else {
split = strings.Split(urlParsed.Path, "/")
}
node := g.tree.getOrSet(split, new(openapi.PathObj), entryWithSource.Id)
opObj, err := handleOpObj(entryWithSource, node.pathObj, g.MaxExampleLen)
if opObj != nil {
return opObj.OperationID, err
}
return "", err
}
func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj, limit int) (*openapi.Operation, error) {
entry := entryWithSource.Entry
isSuccess := 100 <= entry.Response.Status && entry.Response.Status < 400
opObj, wasMissing, err := getOpObj(pathObj, entry.Request.Method, isSuccess)
if err != nil {
return nil, err
}
if !isSuccess && wasMissing {
logger.Log.Debugf("Dropped traffic entry due to failed status and no known endpoint at: %s", entry.StartedDateTime)
return nil, nil
}
err = handleRequest(&entry.Request, opObj, isSuccess, entryWithSource.Id, limit)
if err != nil {
return nil, err
}
err = handleResponse(&entry.Response, opObj, isSuccess, entryWithSource.Id, limit)
if err != nil {
return nil, err
}
err = handleCounters(opObj, isSuccess, entryWithSource)
if err != nil {
return nil, err
}
setSampleID(&opObj.Extensions, entryWithSource.Id)
return opObj, nil
}
func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *EntryWithSource) error {
// TODO: if performance around DecodeExtension+SetExtension is bad, store counters as separate maps
counter := Counter{}
counterMap := CounterMap{}
prevTs := 0.0
if opObj.Extensions == nil {
opObj.Extensions = openapi.Extensions{}
} else {
if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
err := opObj.Extensions.DecodeExtension(CountersTotal, &counter)
if err != nil {
return err
}
}
if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
err := opObj.Extensions.DecodeExtension(CountersPerSource, &counterMap)
if err != nil {
return err
}
}
if _, ok := opObj.Extensions.Extension(LastSeenTS); ok {
err := opObj.Extensions.DecodeExtension(LastSeenTS, &prevTs)
if err != nil {
return err
}
}
}
var counterPerSource *Counter
if existing, ok := counterMap[entryWithSource.Source]; ok {
counterPerSource = existing
} else {
counterPerSource = new(Counter)
counterMap[entryWithSource.Source] = counterPerSource
}
started, err := datetime.Parse(entryWithSource.Entry.StartedDateTime, time.UTC)
if err != nil {
return err
}
ts := float64(started.UnixNano()) / float64(time.Millisecond) / 1000
rt := float64(entryWithSource.Entry.Time) / 1000
dur := 0.0
if prevTs != 0 && ts >= prevTs {
dur = ts - prevTs
}
counter.addEntry(ts, rt, success, dur)
counterPerSource.addEntry(ts, rt, success, dur)
err = opObj.Extensions.SetExtension(LastSeenTS, ts)
if err != nil {
return err
}
err = opObj.Extensions.SetExtension(CountersTotal, counter)
if err != nil {
return err
}
err = opObj.Extensions.SetExtension(CountersPerSource, counterMap)
if err != nil {
return err
}
return nil
}
func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool, sampleId string, limit int) error {
// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
urlParsed, err := url.Parse(req.URL)
if err != nil {
return err
}
qs := make([]har.NVP, 0)
for name, vals := range urlParsed.Query() {
for _, val := range vals {
qs = append(qs, har.NVP{Name: name, Value: val})
}
}
if len(qs) != len(req.QueryString) {
logger.Log.Warningf("QStr params in HAR do not match URL: %s", req.URL)
}
qstrGW := nvParams{
In: openapi.InQuery,
Pairs: qs,
IsIgnored: func(name string) bool { return false },
GeneralizeName: func(name string) string { return name },
}
handleNameVals(qstrGW, &opObj.Parameters, false, sampleId)
hdrGW := nvParams{
In: openapi.InHeader,
Pairs: req.Headers,
IsIgnored: isHeaderIgnored,
GeneralizeName: strings.ToLower,
}
handleNameVals(hdrGW, &opObj.Parameters, true, sampleId)
if isSuccess {
reqBody, err := getRequestBody(req, opObj)
if err != nil {
return err
}
if reqBody != nil {
setSampleID(&reqBody.Extensions, sampleId)
if req.PostData.Text == "" {
reqBody.Required = false
} else {
reqCtype, _ := getReqCtype(req)
reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype, sampleId, limit)
if err != nil {
return err
}
_ = reqMedia
}
}
}
return nil
}
func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool, sampleId string, limit int) error {
// TODO: we don't support "default" response
respObj, err := getResponseObj(resp, opObj, isSuccess)
if err != nil {
return err
}
setSampleID(&respObj.Extensions, sampleId)
handleRespHeaders(resp.Headers, respObj, sampleId)
respCtype := getRespCtype(resp)
respContent := respObj.Content
respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype, sampleId, limit)
if err != nil {
return err
}
_ = respMedia
return nil
}
func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj, sampleId string) {
visited := map[string]*openapi.HeaderObj{}
for _, pair := range reqHeaders {
if isHeaderIgnored(pair.Name) {
continue
}
nameGeneral := strings.ToLower(pair.Name)
initHeaders(respObj)
objHeaders := respObj.Headers
param := findHeaderByName(&respObj.Headers, pair.Name)
if param == nil {
param = createHeader(openapi.TypeString)
objHeaders[nameGeneral] = param
}
exmp := &param.Examples
err := fillParamExample(&exmp, pair.Value)
if err != nil {
logger.Log.Warningf("Failed to add example to a parameter: %s", err)
}
visited[nameGeneral] = param
setSampleID(&param.Extensions, sampleId)
}
// maintain "required" flag
if respObj.Headers != nil {
for name, param := range respObj.Headers {
paramObj, err := param.ResolveHeader(headerResolver)
if err != nil {
logger.Log.Warningf("Failed to resolve param: %s", err)
continue
}
_, ok := visited[strings.ToLower(name)]
if !ok {
flag := false
paramObj.Required = &flag
}
}
}
}
func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, sampleId string, limit int) (*openapi.MediaType, error) {
content, found := respContent[ctype]
if !found {
respContent[ctype] = &openapi.MediaType{}
content = respContent[ctype]
}
setSampleID(&content.Extensions, sampleId)
var text string
var isBinary bool
if reqResp.Req != nil {
isBinary, _, text = reqResp.Req.PostData.B64Decoded()
} else {
isBinary, _, text = reqResp.Resp.Content.B64Decoded()
}
if !isBinary && text != "" {
var exampleMsg []byte
// try treating it as json
anyVal, isJSON := anyJSON(text)
if isJSON {
// re-marshal with forced indent
if msg, err := json.MarshalIndent(anyVal, "", "\t"); err != nil {
panic("Failed to re-marshal value, super-strange")
} else {
exampleMsg = msg
}
} else {
if msg, err := json.Marshal(text); err != nil {
return nil, err
} else {
exampleMsg = msg
}
}
if ctype == "application/x-www-form-urlencoded" && reqResp.Req != nil {
handleFormDataUrlencoded(text, content)
} else if strings.HasPrefix(ctype, "multipart/form-data") && reqResp.Req != nil {
_, params := getReqCtype(reqResp.Req)
handleFormDataMultipart(text, content, params)
}
if len(exampleMsg) > len(content.Example) && (limit < 0 || len(exampleMsg) <= limit) {
content.Example = exampleMsg
}
}
return respContent[ctype], nil
}
func handleFormDataUrlencoded(text string, content *openapi.MediaType) {
formData, err := url.ParseQuery(text)
if err != nil {
logger.Log.Warningf("Could not decode urlencoded: %s", err)
return
}
parts := make([]PartWithBody, 0)
for name, vals := range formData {
for _, val := range vals {
part := new(multipart.Part)
part.Header = textproto.MIMEHeader{}
part.Header.Add("Content-Disposition", "form-data; name=\""+name+"\";")
parts = append(parts, PartWithBody{part: part, body: []byte(val)})
}
}
handleFormData(content, parts)
}
func handleFormData(content *openapi.MediaType, parts []PartWithBody) {
hadSchema := true
if content.Schema == nil {
hadSchema = false // will use it for required flags
content.Schema = new(openapi.SchemaObj)
content.Schema.Type = openapi.Types{openapi.TypeObject}
content.Schema.Properties = openapi.Schemas{}
}
props := &content.Schema.Properties
seenNames := map[string]struct{}{} // set equivalent in Go, yikes
for _, pwb := range parts {
name := pwb.part.FormName()
seenNames[name] = struct{}{}
existing, found := (*props)[name]
if !found {
existing = new(openapi.SchemaObj)
existing.Type = openapi.Types{openapi.TypeString}
(*props)[name] = existing
ctype := pwb.part.Header.Get("content-type")
if ctype != "" {
if existing.Keywords == nil {
existing.Keywords = map[string]json.RawMessage{}
}
existing.Keywords["contentMediaType"], _ = json.Marshal(ctype)
}
}
addSchemaExample(existing, string(pwb.body))
}
// handle required flag
if content.Schema.Required == nil {
if !hadSchema {
content.Schema.Required = make([]string, 0)
for name := range seenNames {
content.Schema.Required = append(content.Schema.Required, name)
}
sort.Strings(content.Schema.Required)
} // else it's a known schema with no required fields
} else {
content.Schema.Required = intersectSliceWithMap(content.Schema.Required, seenNames)
sort.Strings(content.Schema.Required)
}
}
type PartWithBody struct {
part *multipart.Part
body []byte
}
func handleFormDataMultipart(text string, content *openapi.MediaType, ctypeParams map[string]string) {
boundary, ok := ctypeParams["boundary"]
if !ok {
logger.Log.Errorf("Multipart header has no boundary")
return
}
mpr := multipart.NewReader(strings.NewReader(text), boundary)
parts := make([]PartWithBody, 0)
for {
part, err := mpr.NextPart()
if err == io.EOF {
break
}
if err != nil {
logger.Log.Errorf("Cannot parse multipart body: %v", err)
break
}
defer part.Close()
body, err := ioutil.ReadAll(part)
if err != nil {
logger.Log.Errorf("Error reading multipart Part %s: %v", part.Header, err)
}
parts = append(parts, PartWithBody{part: part, body: body})
}
handleFormData(content, parts)
}
func getRespCtype(resp *har.Response) string {
var ctype string
ctype = resp.Content.MimeType
for _, hdr := range resp.Headers {
if strings.ToLower(hdr.Name) == "content-type" {
ctype = hdr.Value
}
}
mediaType, _, err := mime.ParseMediaType(ctype)
if err != nil {
return ""
}
return mediaType
}
func getReqCtype(req *har.Request) (ctype string, params map[string]string) {
ctype = req.PostData.MimeType
for _, hdr := range req.Headers {
if strings.ToLower(hdr.Name) == "content-type" {
ctype = hdr.Value
}
}
if ctype == "" {
return "", map[string]string{}
}
mediaType, params, err := mime.ParseMediaType(ctype)
if err != nil {
logger.Log.Errorf("Cannot parse Content-Type header %q: %v", ctype, err)
return "", map[string]string{}
}
return mediaType, params
}
func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool) (*openapi.ResponseObj, error) {
statusStr := strconv.Itoa(resp.Status)
var response openapi.Response
response, found := opObj.Responses[statusStr]
if !found {
if opObj.Responses == nil {
opObj.Responses = map[string]openapi.Response{}
}
opObj.Responses[statusStr] = &openapi.ResponseObj{Content: map[string]*openapi.MediaType{}}
response = opObj.Responses[statusStr]
}
resResponse, err := response.ResolveResponse(responseResolver)
if err != nil {
return nil, err
}
if isSuccess {
resResponse.Description = "Successful call with status " + statusStr
} else {
resResponse.Description = "Failed call with status " + statusStr
}
return resResponse, nil
}
func getRequestBody(req *har.Request, opObj *openapi.Operation) (*openapi.RequestBodyObj, error) {
if opObj.RequestBody == nil {
// create if there is body in request
if req.PostData.Text != "" {
opObj.RequestBody = &openapi.RequestBodyObj{Description: "Generic request body", Required: true, Content: map[string]*openapi.MediaType{}}
} else {
return nil, nil
}
}
reqBody, err := opObj.RequestBody.ResolveRequestBody(reqBodyResolver)
if err != nil {
return nil, err
}
return reqBody, nil
}
func getOpObj(pathObj *openapi.PathObj, method string, createIfNone bool) (*openapi.Operation, bool, error) {
method = strings.ToLower(method)
var op **openapi.Operation
switch method {
case "get":
op = &pathObj.Get
case "put":
op = &pathObj.Put
case "post":
op = &pathObj.Post
case "delete":
op = &pathObj.Delete
case "options":
op = &pathObj.Options
case "head":
op = &pathObj.Head
case "patch":
op = &pathObj.Patch
case "trace":
op = &pathObj.Trace
default:
return nil, false, errors.New("unsupported HTTP method: " + method)
}
isMissing := false
if *op == nil {
isMissing = true
if createIfNone {
*op = &openapi.Operation{Responses: map[string]openapi.Response{}}
newUUID := uuid.New().String()
(**op).OperationID = newUUID
} else {
return nil, isMissing, nil
}
}
return *op, isMissing, nil
}

View File

@ -0,0 +1,267 @@
package oas
import (
"encoding/json"
"io/ioutil"
"os"
"regexp"
"strings"
"sync"
"testing"
"time"
"github.com/chanced/openapi"
"github.com/up9inc/mizu/agent/pkg/har"
"github.com/up9inc/mizu/logger"
"github.com/wI2L/jsondiff"
)
// if started via env, write file into subdir
func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) string {
content, err := json.MarshalIndent(spec, "", " ")
if err != nil {
panic(err)
}
if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
path := "./oas-samples"
err := os.MkdirAll(path, 0o755)
if err != nil {
panic(err)
}
err = ioutil.WriteFile(path+"/"+label+".json", content, 0644)
if err != nil {
panic(err)
}
t.Logf("Written: %s", label)
} else {
t.Logf("%s", string(content))
}
return string(content)
}
func TestEntries(t *testing.T) {
//logger.InitLoggerStd(logging.INFO) causes race condition
files, err := getFiles("./test_artifacts/")
if err != nil {
t.Log(err)
t.FailNow()
}
gen := NewDefaultOasGenerator(-1)
gen.serviceSpecs = new(sync.Map)
loadStartingOAS("test_artifacts/catalogue.json", "catalogue", gen.serviceSpecs)
loadStartingOAS("test_artifacts/trcc.json", "trcc-api-service", gen.serviceSpecs)
go func() {
for {
time.Sleep(1 * time.Second)
gen.serviceSpecs.Range(func(key, val interface{}) bool {
svc := key.(string)
t.Logf("Getting spec for %s", svc)
gen := val.(*SpecGen)
_, err := gen.GetSpec()
if err != nil {
t.Error(err)
}
return true
})
}
}()
cnt, err := feedEntries(files, true, gen)
if err != nil {
t.Log(err)
t.Fail()
}
svcs := strings.Builder{}
gen.serviceSpecs.Range(func(key, val interface{}) bool {
gen := val.(*SpecGen)
svc := key.(string)
svcs.WriteString(svc + ",")
spec, err := gen.GetSpec()
if err != nil {
t.Log(err)
t.FailNow()
return false
}
err = spec.Validate()
if err != nil {
specText, _ := json.MarshalIndent(spec, "", "\t")
t.Log(string(specText))
t.Log(err)
t.FailNow()
}
return true
})
gen.serviceSpecs.Range(func(key, val interface{}) bool {
svc := key.(string)
gen := val.(*SpecGen)
spec, err := gen.GetSpec()
if err != nil {
t.Log(err)
t.FailNow()
}
outputSpec(svc, spec, t)
err = spec.Validate()
if err != nil {
t.Log(err)
t.FailNow()
}
return true
})
logger.Log.Infof("Total entries: %d", cnt)
}
func TestFileSingle(t *testing.T) {
gen := NewDefaultOasGenerator(-1)
gen.serviceSpecs = new(sync.Map)
// loadStartingOAS()
file := "test_artifacts/params.har"
files := []string{file}
cnt, err := feedEntries(files, true, gen)
if err != nil {
logger.Log.Warning("Failed processing file: " + err.Error())
t.Fail()
}
gen.serviceSpecs.Range(func(key, val interface{}) bool {
svc := key.(string)
gen := val.(*SpecGen)
spec, err := gen.GetSpec()
if err != nil {
t.Log(err)
t.FailNow()
}
specText := outputSpec(svc, spec, t)
err = spec.Validate()
if err != nil {
t.Log(err)
t.FailNow()
}
expected, err := ioutil.ReadFile(file + ".spec.json")
if err != nil {
t.Errorf(err.Error())
t.FailNow()
}
patFloatPrecision := regexp.MustCompile(`(\d+\.\d{1,2})(\d*)`)
expected = []byte(patUuid4.ReplaceAllString(string(expected), "<UUID4>"))
specText = patUuid4.ReplaceAllString(specText, "<UUID4>")
expected = []byte(patFloatPrecision.ReplaceAllString(string(expected), "$1"))
specText = patFloatPrecision.ReplaceAllString(specText, "$1")
diff, err := jsondiff.CompareJSON(expected, []byte(specText))
if err != nil {
t.Errorf(err.Error())
t.FailNow()
}
if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
err = ioutil.WriteFile(file+".spec.json", []byte(specText), 0644)
if err != nil {
panic(err)
}
}
if len(diff) > 0 {
t.Errorf("Generated spec does not match expected:\n%s", diff.String())
}
return true
})
logger.Log.Infof("Processed entries: %d", cnt)
}
func loadStartingOAS(file string, label string, specs *sync.Map) {
fd, err := os.Open(file)
if err != nil {
panic(err)
}
defer fd.Close()
data, err := ioutil.ReadAll(fd)
if err != nil {
panic(err)
}
var doc *openapi.OpenAPI
err = json.Unmarshal(data, &doc)
if err != nil {
panic(err)
}
gen := NewGen(label)
gen.StartFromSpec(doc)
specs.Store(label, gen)
}
func TestEntriesNegative(t *testing.T) {
gen := NewDefaultOasGenerator(-1)
gen.serviceSpecs = new(sync.Map)
files := []string{"invalid"}
_, err := feedEntries(files, false, gen)
if err == nil {
t.Logf("Should have failed")
t.Fail()
}
}
func TestEntriesPositive(t *testing.T) {
gen := NewDefaultOasGenerator(-1)
gen.serviceSpecs = new(sync.Map)
files := []string{"test_artifacts/params.har"}
_, err := feedEntries(files, false, gen)
if err != nil {
t.Logf("Failed")
t.Fail()
}
}
func TestLoadValidHAR(t *testing.T) {
inp := `{"startedDateTime": "2021-02-03T07:48:12.959000+00:00", "time": 1, "request": {"method": "GET", "url": "http://unresolved_target/1.0.0/health", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "queryString": [], "headersSize": -1, "bodySize": -1}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "content": {"size": 2, "mimeType": "", "text": "OK"}, "redirectURL": "", "headersSize": -1, "bodySize": 2}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 1}}`
var entry *har.Entry
var err = json.Unmarshal([]byte(inp), &entry)
if err != nil {
t.Logf("Failed to decode entry: %s", err)
t.FailNow() // demonstrates the problem of `martian` HAR library
}
}
func TestLoadValid3_1(t *testing.T) {
fd, err := os.Open("test_artifacts/catalogue.json")
if err != nil {
t.Log(err)
t.FailNow()
}
defer fd.Close()
data, err := ioutil.ReadAll(fd)
if err != nil {
t.Log(err)
t.FailNow()
}
var oas openapi.OpenAPI
err = json.Unmarshal(data, &oas)
if err != nil {
t.Log(err)
t.FailNow()
}
}

View File

@ -0,0 +1,51 @@
{
"openapi": "3.1.0",
"info": {
"title": "Preloaded",
"version": "0.1",
"description": "Test file for loading pre-existing OAS"
},
"paths": {
"/catalogue/{id}": {
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"example": "some-uuid-maybe"
}
],
"get": {
"parameters": [ {
"name": "non-required-header",
"in": "header",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"example": "some-uuid-maybe"
}
]
}
},
"/catalogue/{id}/details": {
"parameters": [
{
"name": "id",
"in": "path",
"style": "simple",
"required": true,
"schema": {
"type": "string"
},
"example": "some-uuid-maybe"
}
]
}
}
}

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,13 @@
{"messageType": "http", "_source": "some-source", ",firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78415179Z", "time": 13, "request": {"method": "GET", "url": "http://catalogue/catalogue/size?tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "x-some", "value": "demo val"},{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [{"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "22"}], "content": {"size": 22, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInNpemUiOjl9", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 22}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 13}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:17.784918698Z", "time": 19, "request": {"method": "GET", "url": "http://catalogue/catalogue?page=1&size=6&tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "page", "value": "1"}, {"name": "size", "value": "6"}, {"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "1927"}], "content": {"size": 1927, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19LHsiaWQiOiI4MzdhYjE0MS0zOTllLTRjMWYtOWFiYy1iYWNlNDAyOTZiYWMiLCJuYW1lIjoiQ2F0IHNvY2tzIiwiZGVzY3JpcHRpb24iOiJjb25zZXF1YXQgYW1ldCBjdXBpZGF0YXQgbWluaW0gbGFib3J1bSB0ZW1wb3IgZWxpdCBleCBjb25zZXF1YXQgaW4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2NrczIuanBnIl0sInByaWNlIjoxNSwiY291bnQiOjE3NSwidGFnIjpbImJyb3duIiwiZm9ybWFsIiwiZ3JlZW4iXX1dCg==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1927}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 19}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78418182Z", "time": 7, "request": {"method": "GET", "url": "http://catalogue/tags", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "107"}], "content": {"size": 107, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInRhZ3MiOlsiYnJvd24iLCJnZWVrIiwiZm9ybWFsIiwiYmx1ZSIsInNraW4iLCJyZWQiLCJhY3Rpb24iLCJzcG9ydCIsImJsYWNrIiwibWFnaWMiLCJncmVlbiJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 107}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 7}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:18.131501482Z", "time": 5, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}, {"name": "x-some", "value": "demoval"}], ",queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 5}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:18.379836908Z", "time": 14, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 14}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:22.920540124Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/808a2de1-1aaa-4c25-a9b9-6612e8f29a38", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "275"}], "content": {"size": 275, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NzM4LCJkZXNjcmlwdGlvbiI6IkEgbWF0dXJlIHNvY2ssIGNyb3NzZWQsIHdpdGggYW4gYWlyIG9mIG5vbmNoYWxhbmNlLiIsImlkIjoiODA4YTJkZTEtMWFhYS00YzI1LWE5YjktNjYxMmU4ZjI5YTM4IiwiaW1hZ2VVcmwiOlsiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMS5qcGVnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMi5qcGVnIl0sIm5hbWUiOiJDcm9zc2VkIiwicHJpY2UiOjE3LjMyLCJ0YWciOlsiYmx1ZSIsInJlZCIsImFjdGlvbiIsImZvcm1hbCJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 275}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:22.921609501Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?sort=id&size=3&tags=blue", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "3"}, {"name": "tags", "value": "blue"}, {"name": "sort", "value": "id"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "789"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}], "content": {"size": 789, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJuYW1lIjoiQ29sb3VyZnVsIiwiZGVzY3JpcHRpb24iOiJwcm9pZGVudCBvY2NhZWNhdCBpcnVyZSBldCBleGNlcHRldXIgbGFib3JlIG1pbmltIG5pc2kgYW1ldCBpcnVyZSIsImltYWdlVXJsIjpbIi9jYXRhbG9ndWUvaW1hZ2VzL2NvbG91cmZ1bF9zb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIl0sInByaWNlIjoxOCwiY291bnQiOjQzOCwidGFnIjpbImJsdWUiXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiYmx1ZSJdfV0K", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 789}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:22.923197848Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:23.175549218Z", "time": 26, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 26}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:25.239777333Z", "time": 10, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Transfer-Encoding", "value": "chunked"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "X-Application-Context", "value": "carts:80"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 10}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:25.725866772Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?size=5", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "5"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "1643"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}], "content": {"size": 1643, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19XQo=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1643}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:25.729303217Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}

View File

@ -0,0 +1,790 @@
{
"log": {
"version": "1.2",
"creator": {
"name": "mitmproxy har_dump",
"version": "0.1",
"comment": "mitmproxy version mitmproxy 4.0.4"
},
"entries": [
{
"startedDateTime": "2019-09-06T06:14:43.864529+00:00",
"time": 111,
"request": {
"method": "GET",
"url": "https://httpbin.org/e21f7112-3d3b-4632-9da3-a4af2e0e9166/sub1",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"headersSize": 1542,
"bodySize": 0,
"queryString": []
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [
],
"content": {
"mimeType": "text/html",
"text": "",
"size": 0
},
"redirectURL": "",
"headersSize": 245,
"bodySize": 39
},
"cache": {},
"timings": {
"send": 22,
"receive": 2,
"wait": 87,
"connect": -1,
"ssl": -1
},
"serverIPAddress": "54.210.29.33"
},
{
"startedDateTime": "2019-09-06T06:16:18.747122+00:00",
"time": 630,
"request": {
"method": "GET",
"url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a/sub2",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"queryString": [],
"headersSize": 1542,
"bodySize": 0
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"content": {
"size": 39,
"compression": -20,
"mimeType": "application/json",
"text": "null"
},
"redirectURL": "",
"headersSize": 248,
"bodySize": 39
},
"cache": {},
"timings": {
"send": 14,
"receive": 4,
"wait": 350,
"connect": 262,
"ssl": -1
}
},
{
"startedDateTime": "2019-09-06T06:16:19.747122+00:00",
"time": 630,
"request": {
"method": "GET",
"url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a;mparam=matrixparam",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"queryString": [],
"headersSize": 1542,
"bodySize": 0
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"content": {
"size": 39,
"compression": -20,
"mimeType": "application/json",
"text": "null"
},
"redirectURL": "",
"headersSize": 248,
"bodySize": 39
},
"cache": {},
"timings": {
"send": 14,
"receive": 4,
"wait": 350,
"connect": 262,
"ssl": -1
}
},
{
"startedDateTime": "2019-09-06T06:16:20.047122+00:00",
"time": 630,
"request": {
"method": "GET",
"url": "https://httpbin.org/appears-once",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"queryString": [],
"headersSize": 1542,
"bodySize": 0
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"content": {
"size": 39,
"compression": -20,
"mimeType": "application/json",
"text": "null"
},
"redirectURL": "",
"headersSize": 248,
"bodySize": 39
},
"cache": {},
"timings": {
"send": 14,
"receive": 4,
"wait": 350,
"connect": 262,
"ssl": -1
}
},
{
"startedDateTime": "2019-09-06T06:16:20.747122+00:00",
"time": 630,
"request": {
"method": "GET",
"url": "https://httpbin.org/appears-twice",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"queryString": [],
"headersSize": 1542,
"bodySize": 0
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"content": {
"size": 39,
"compression": -20,
"mimeType": "application/json",
"text": "null"
},
"redirectURL": "",
"headersSize": 248,
"bodySize": 39
},
"cache": {},
"timings": {
"send": 14,
"receive": 4,
"wait": 350,
"connect": 262,
"ssl": -1
}
},
{
"startedDateTime": "2019-09-06T06:16:21.747122+00:00",
"time": 630,
"request": {
"method": "GET",
"url": "https://httpbin.org/appears-twice",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"queryString": [],
"headersSize": 1542,
"bodySize": 0
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"content": {
"size": 39,
"compression": -20,
"mimeType": "application/json",
"text": "null"
},
"redirectURL": "",
"headersSize": 248,
"bodySize": 39
},
"cache": {},
"timings": {
"send": 14,
"receive": 4,
"wait": 350,
"connect": 262,
"ssl": -1
}
},
{
"startedDateTime": "2019-09-06T06:16:20.747122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/form-urlencoded",
"httpVersion": "",
"cookies": [],
"headers": [
{
"name": "Content-Type",
"value": "application/x-www-form-urlencoded"
}
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": "agent-id=ade&callback-url=&token=sometoken"
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:21.747122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/form-urlencoded",
"httpVersion": "",
"cookies": [],
"headers": [
{
"name": "Content-Type",
"value": "application/x-www-form-urlencoded"
}
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": "agent-id=ade&callback-url=&token=sometoken-second-val&optional=another"
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:22.747122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/form-multipart",
"httpVersion": "",
"cookies": [],
"headers": [
{
"name": "Content-Type",
"value": "multipart/form-data; boundary=BOUNDARY"
}
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n"
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 62,
"mimeType": "",
"text": "{}"
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 62
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:21.757122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/body-optional",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:21.747122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/body-optional",
"httpVersion": "",
"cookies": [],
"headers": [
{
"name": "Content-Type",
"value": "application/json"
}
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": "{\"key\", \"val\"}"
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:21.757122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/body-optional",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:21.757122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/body-required",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": "body exists"
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:22.000000+00:00",
"time": 1,
"request": {
"method": "GET",
"url": "https://httpbin.org/param-patterns/prefix-gibberish-fine/234324",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:22.000001+00:00",
"time": 1,
"request": {
"method": "GET",
"url": "https://httpbin.org/param-patterns/prefix-gibberish-sfdlasdfkadf87sd93284q24r/1",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:22.000002+00:00",
"time": 1,
"request": {
"method": "GET",
"url": "https://httpbin.org/param-patterns/prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf/static",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:22.000003+00:00",
"time": 1,
"request": {
"method": "GET",
"url": "https://httpbin.org/param-patterns/prefix-gibberish-4jk5l2345h2452l4352435jlk45",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:22.000004+00:00",
"time": 1,
"request": {
"method": "GET",
"url": "https://httpbin.org/param-patterns/prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:22.000002+00:00",
"time": 1,
"request": {
"method": "GET",
"url": "https://httpbin.org/param-patterns/prefix-gibberish-afterwards/23421",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
}
]
}
}

View File

@ -0,0 +1,897 @@
{
"openapi": "3.1.0",
"info": {
"title": "https://httpbin.org",
"description": "Mizu observed 19 entries (0 failed), at 0.10 hits/s, average response time is 0.17 seconds",
"version": "1.0"
},
"servers": [
{
"url": "https://httpbin.org"
}
],
"paths": {
"/appears-once": {
"get": {
"summary": "/appears-once",
"description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000004"
}
},
"x-sample-entry": "000000000000000000000004"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750580.04,
"lastSeen": 1567750580.04,
"sumRT": 0.63,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750580.04,
"lastSeen": 1567750580.04,
"sumRT": 0.63,
"sumDuration": 0
},
"x-last-seen-ts": 1567750580.04,
"x-sample-entry": "000000000000000000000004"
}
},
"/appears-twice": {
"get": {
"summary": "/appears-twice",
"description": "Mizu observed 2 entries (0 failed), at 0.50 hits/s, average response time is 0.63 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000006"
}
},
"x-sample-entry": "000000000000000000000006"
}
},
"x-counters-per-source": {
"": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.74,
"lastSeen": 1567750581.74,
"sumRT": 1.26,
"sumDuration": 1
}
},
"x-counters-total": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.74,
"lastSeen": 1567750581.74,
"sumRT": 1.26,
"sumDuration": 1
},
"x-last-seen-ts": 1567750581.74,
"x-sample-entry": "000000000000000000000006"
}
},
"/body-optional": {
"post": {
"summary": "/body-optional",
"description": "Mizu observed 3 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000012"
}
},
"x-sample-entry": "000000000000000000000012"
}
},
"x-counters-per-source": {
"": {
"entries": 3,
"failures": 0,
"firstSeen": 1567750581.74,
"lastSeen": 1567750581.75,
"sumRT": 0.00,
"sumDuration": 0.01
}
},
"x-counters-total": {
"entries": 3,
"failures": 0,
"firstSeen": 1567750581.74,
"lastSeen": 1567750581.75,
"sumRT": 0.00,
"sumDuration": 0.01
},
"x-last-seen-ts": 1567750581.75,
"x-sample-entry": "000000000000000000000012",
"requestBody": {
"description": "Generic request body",
"content": {
"application/json": {
"example": "{\"key\", \"val\"}",
"x-sample-entry": "000000000000000000000011"
}
},
"x-sample-entry": "000000000000000000000012"
}
}
},
"/body-required": {
"post": {
"summary": "/body-required",
"description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000013"
}
},
"x-sample-entry": "000000000000000000000013"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750581.75,
"lastSeen": 1567750581.75,
"sumRT": 0.00,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750581.75,
"lastSeen": 1567750581.75,
"sumRT": 0.00,
"sumDuration": 0
},
"x-last-seen-ts": 1567750581.75,
"x-sample-entry": "000000000000000000000013",
"requestBody": {
"description": "Generic request body",
"content": {
"": {
"example": "body exists",
"x-sample-entry": "000000000000000000000013"
}
},
"required": true,
"x-sample-entry": "000000000000000000000013"
}
}
},
"/form-multipart": {
"post": {
"summary": "/form-multipart",
"description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"example": {},
"x-sample-entry": "000000000000000000000009"
}
},
"x-sample-entry": "000000000000000000000009"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.74,
"lastSeen": 1567750582.74,
"sumRT": 0.00,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.74,
"lastSeen": 1567750582.74,
"sumRT": 0.00,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.74,
"x-sample-entry": "000000000000000000000009",
"requestBody": {
"description": "Generic request body",
"content": {
"multipart/form-data": {
"schema": {
"type": "object",
"required": [
"file",
"path"
],
"properties": {
"file": {
"type": "string",
"contentMediaType": "application/json",
"examples": [
"{\"functions\": 123}"
]
},
"path": {
"type": "string",
"examples": [
"/content/components"
]
}
}
},
"example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n",
"x-sample-entry": "000000000000000000000009"
}
},
"required": true,
"x-sample-entry": "000000000000000000000009"
}
}
},
"/form-urlencoded": {
"post": {
"summary": "/form-urlencoded",
"description": "Mizu observed 2 entries (0 failed), at 0.50 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000008"
}
},
"x-sample-entry": "000000000000000000000008"
}
},
"x-counters-per-source": {
"": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.74,
"lastSeen": 1567750581.74,
"sumRT": 0.00,
"sumDuration": 1
}
},
"x-counters-total": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.74,
"lastSeen": 1567750581.74,
"sumRT": 0.00,
"sumDuration": 1
},
"x-last-seen-ts": 1567750581.74,
"x-sample-entry": "000000000000000000000008",
"requestBody": {
"description": "Generic request body",
"content": {
"application/x-www-form-urlencoded": {
"schema": {
"type": "object",
"required": [
"agent-id",
"callback-url",
"token"
],
"properties": {
"agent-id": {
"type": "string",
"examples": [
"ade"
]
},
"callback-url": {
"type": "string",
"examples": [
""
]
},
"optional": {
"type": "string",
"examples": [
"another"
]
},
"token": {
"type": "string",
"examples": [
"sometoken",
"sometoken-second-val"
]
}
}
},
"example": "agent-id=ade\u0026callback-url=\u0026token=sometoken-second-val\u0026optional=another",
"x-sample-entry": "000000000000000000000008"
}
},
"required": true,
"x-sample-entry": "000000000000000000000008"
}
}
},
"/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}",
"description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000014"
}
},
"x-sample-entry": "000000000000000000000014"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582,
"lastSeen": 1567750582,
"sumRT": 0.00,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582,
"lastSeen": 1567750582,
"sumRT": 0.00,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582,
"x-sample-entry": "000000000000000000000014"
},
"parameters": [
{
"name": "prefixgibberishfineId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "234324"
}
},
"x-sample-entry": "000000000000000000000014"
}
]
},
"/param-patterns/{parampatternId}": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}",
"description": "Mizu observed 2 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000018"
}
},
"x-sample-entry": "000000000000000000000018"
}
},
"x-counters-per-source": {
"": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 9.53e-7
}
},
"x-counters-total": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 9.53e-7
},
"x-last-seen-ts": 1567750582.00,
"x-sample-entry": "000000000000000000000018"
},
"parameters": [
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/param-patterns/{parampatternId}/1": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}/1",
"description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000015"
}
},
"x-sample-entry": "000000000000000000000015"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.00,
"x-sample-entry": "000000000000000000000015"
},
"parameters": [
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/param-patterns/{parampatternId}/static": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}/static",
"description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000016"
}
},
"x-sample-entry": "000000000000000000000016"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.00,
"x-sample-entry": "000000000000000000000016"
},
"parameters": [
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/param-patterns/{parampatternId}/{param1}": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}/{param1}",
"description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000019"
}
},
"x-sample-entry": "000000000000000000000019"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.00,
"x-sample-entry": "000000000000000000000019"
},
"parameters": [
{
"name": "param1",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "23421"
}
},
"x-sample-entry": "000000000000000000000019"
},
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/{Id}": {
"get": {
"summary": "/{Id}",
"description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000003"
}
},
"x-sample-entry": "000000000000000000000003"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750579.74,
"lastSeen": 1567750579.74,
"sumRT": 0.63,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750579.74,
"lastSeen": 1567750579.74,
"sumRT": 0.63,
"sumDuration": 0
},
"x-last-seen-ts": 1567750579.74,
"x-sample-entry": "000000000000000000000003"
},
"parameters": [
{
"name": "Id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "<UUID4>"
},
"example #1": {
"value": "<UUID4>"
}
},
"x-sample-entry": "000000000000000000000003"
}
]
},
"/{Id}/sub1": {
"get": {
"summary": "/{Id}/sub1",
"description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.11 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"text/html": {
"x-sample-entry": "000000000000000000000001"
}
},
"x-sample-entry": "000000000000000000000001"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750483.86,
"lastSeen": 1567750483.86,
"sumRT": 0.11,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750483.86,
"lastSeen": 1567750483.86,
"sumRT": 0.11,
"sumDuration": 0
},
"x-last-seen-ts": 1567750483.86,
"x-sample-entry": "000000000000000000000001"
},
"parameters": [
{
"name": "Id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "<UUID4>"
},
"example #1": {
"value": "<UUID4>"
}
},
"x-sample-entry": "000000000000000000000003"
}
]
},
"/{Id}/sub2": {
"get": {
"summary": "/{Id}/sub2",
"description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000002"
}
},
"x-sample-entry": "000000000000000000000002"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750578.74,
"lastSeen": 1567750578.74,
"sumRT": 0.63,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750578.74,
"lastSeen": 1567750578.74,
"sumRT": 0.63,
"sumDuration": 0
},
"x-last-seen-ts": 1567750578.74,
"x-sample-entry": "000000000000000000000002"
},
"parameters": [
{
"name": "Id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "<UUID4>"
},
"example #1": {
"value": "<UUID4>"
}
},
"x-sample-entry": "000000000000000000000003"
}
]
}
},
"x-counters-per-source": {
"": {
"entries": 19,
"failures": 0,
"firstSeen": 1567750483.86,
"lastSeen": 1567750582.74,
"sumRT": 3.27,
"sumDuration": 2.01
}
},
"x-counters-total": {
"entries": 19,
"failures": 0,
"firstSeen": 1567750483.86,
"lastSeen": 1567750582.74,
"sumRT": 3.27,
"sumDuration": 2.01
}
}

Some files were not shown because too many files have changed in this diff Show More