github/kubeshark
1
0
Fork 0
mirror of https://github.com/kubeshark/kubeshark.git synced 2025-06-23 14:58:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #182 from up9inc/develop

Browse Source 
Release 2021-08-08
This commit is contained in:
Igor Gov 2021-08-08 14:50:30 +03:00 committed by GitHub
commit e61871a68e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
126 changed files with 6007 additions and 1480 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/publish.yml vendored
View File

@ -66,6 +66,9 @@ jobs:
COMMIT_HASH=${{ github.sha }}
- name: Build and Push CLI
run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
- shell: bash
run: |
echo '${{ steps.versioning.outputs.version }}' >> cli/bin/version.txt
- name: publish
uses: ncipollo/release-action@v1
with:
@ -75,3 +78,4 @@ jobs:
tag: ${{ steps.versioning.outputs.version }}
prerelease: ${{ github.ref != 'refs/heads/main' }}
bodyFile: 'cli/bin/README.md'

17
Dockerfile
View File

@ -13,10 +13,10 @@ ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
RUN apk add libpcap-dev gcc g++ make
# Move to api working directory (/api-build).
WORKDIR /app/api-build
# Move to agent working directory (/agent-build).
WORKDIR /app/agent-build
COPY api/go.mod api/go.sum ./
COPY agent/go.mod agent/go.sum ./
COPY shared/go.mod shared/go.mod ../shared/
COPY tap/go.mod tap/go.mod ../tap/
RUN go mod download
@ -28,10 +28,10 @@ ARG GIT_BRANCH
ARG BUILD_TIMESTAMP
ARG SEM_VER
# Copy and build api code
# Copy and build agent code
COPY shared ../shared
COPY tap ../tap
COPY api .
COPY agent .
RUN go build -ldflags="-s -w \
-X 'mizuserver/pkg/version.GitCommitHash=${COMMIT_HASH}' \
-X 'mizuserver/pkg/version.Branch=${GIT_BRANCH}' \
@ -45,10 +45,13 @@ RUN apk add bash libpcap-dev tcpdump
WORKDIR /app
# Copy binary and config files from /build to root folder of scratch container.
COPY --from=builder ["/app/api-build/mizuagent", "."]
COPY --from=builder ["/app/agent-build/mizuagent", "."]
COPY --from=site-build ["/app/ui-build/build", "site"]
COPY api/start.sh .
COPY agent/start.sh .
# gin-gonic runs in debug mode without this
ENV GIN_MODE=release
# this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
ENTRYPOINT "/app/mizuagent"

42
Makefile
View File

@ -8,7 +8,7 @@ SHELL=/bin/bash
# HELP
# This will output the help for each task
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY: help ui api cli tap docker
.PHONY: help ui agent cli tap docker
help: ## This help.
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@ -19,34 +19,30 @@ help: ## This help.
TS_SUFFIX="$(shell date '+%s')"
GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
export SEM_VER?=0.0.0
ui: ## build UI
ui: ## Build UI.
@(cd ui; npm i ; npm run build; )
@ls -l ui/build
cli: # build CLI
cli: ## Build CLI.
@echo "building cli"; cd cli && $(MAKE) build
api: ## build API server
@(echo "building API server .." )
@(cd api; go build -o build/apiserver main.go)
@ls -l api/build
agent: ## Build agent.
@(echo "building mizu agent .." )
@(cd agent; go build -o build/mizuagent main.go)
@ls -l agent/build
#tap: ## build tap binary
# @(cd tap; go build -o build/tap ./src)
# @ls -l tap/build
docker: ## Build and publish agent docker image.
$(MAKE) push-docker
docker: ## build Docker image
@(echo "building docker image" )
./build-push-featurebranch.sh
push: push-docker push-cli ## Build and publish agent docker image & CLI.
push: push-docker push-cli ## build and publish Mizu docker image & CLI
push-docker:
push-docker: ## Build and publish agent docker image.
@echo "publishing Docker image .. "
./build-push-featurebranch.sh
push-cli:
push-cli: ## Build and publish CLI.
@echo "publishing CLI .. "
@cd cli; $(MAKE) build-all
@echo "publishing file ${OUTPUT_FILE} .."
@ -55,17 +51,17 @@ push-cli:
gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
clean: clean-ui clean-api clean-cli clean-docker ## Clean all build artifacts
clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
clean-ui:
clean-ui: ## Clean UI.
@(rm -rf ui/build ; echo "UI cleanup done" )
clean-api:
@(rm -rf api/build ; echo "api cleanup done" )
clean-agent: ## Clean agent.
@(rm -rf agent/build ; echo "agent cleanup done" )
clean-cli:
clean-cli: ## Clean CLI.
@(cd cli; make clean ; echo "CLI cleanup done" )
clean-docker:
clean-docker:
@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )

328
PERMISSIONS.md Normal file
View File

@ -0,0 +1,328 @@
![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
# Kubernetes permissions for MIZU
This document describes in details all permissions required for full and correct operation of Mizu
We broke down this list into few categories:
- Required - what is needed for `mizu` to run properly on your k8s cluster
- Optional - permissions needed for proper name resolving for service & pod IPs
- addition required for policy validation
# Required permissions
Mizu needs following permissions on your Kubernetes cluster to run properly
```yaml
- apiGroups:
- ""
resources:
- pods
verbs:
- list
- watch
- create
- delete
- apiGroups:
- ""
resources:
- services
verbs:
- create
- delete
- apiGroups:
- apps
resources:
- daemonsets
verbs:
- create
- patch
- delete
- apiGroups:
- ""
resources:
- namespaces
verbs:
- get
- list
- watch
- create
- delete
- apiGroups:
- ""
resources:
- services/proxy
verbs:
- get
```
## Permissions required for service / pod name resolving (opt)
Optionally, for proper resolving of IP addresses to Kubernetes service name, Mizu needs below permissions:
```yaml
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- list
- watch
- create
- delete
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- create
- delete
- apiGroups:
- apps
resources:
- daemonsets
verbs:
- create
- patch
- delete
- apiGroups:
- ""
resources:
- namespaces
verbs:
- get
- list
- watch
- create
- delete
- apiGroups:
- ""
resources:
- services/proxy
verbs:
- get
- apiGroups:
- ""
resources:
- serviceaccounts
verbs:
- get
- create
- delete
- apiGroups:
- rbac.authorization.k8s.io
resources:
- clusterroles
verbs:
- get
- create
- delete
- apiGroups:
- rbac.authorization.k8s.io
resources:
- clusterrolebindings
verbs:
- get
- create
- delete
- apiGroups:
- rbac.authorization.k8s.io
resources:
- roles
verbs:
- get
- create
- delete
- apiGroups:
- rbac.authorization.k8s.io
resources:
- rolebindings
verbs:
- get
- create
- delete
- apiGroups:
- apps
- extensions
resources:
- pods
verbs:
- get
- list
- watch
- apiGroups:
- apps
- extensions
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- ""
- apps
- extensions
resources:
- endpoints
verbs:
- get
- list
- watch
```
## Permissions for Policy rules validation feature (opt)
Optionally, in order to use the policy rules validation feature, Mizu requires the following additional permissions:
```yaml
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- create
- delete
```
- - -
## Namespace-Restricted mode
Alternatively, in order to restrict Mizu to one namespace only (by setting `agent.namespace` in the config file), Mizu needs the following permissions in that namespace:
```yaml
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- list
- watch
- create
- delete
- apiGroups:
- ""
resources:
- services
verbs:
- get
- create
- delete
- apiGroups:
- apps
resources:
- daemonsets
verbs:
- get
- create
- patch
- delete
- apiGroups:
- ""
resources:
- services/proxy
verbs:
- get
```
### Name resolving in Namespace-Restricted mode (opt)
To restrict Mizu to one namespace while also resolving IPs, Mizu needs the following permissions in that namespace:
```yaml
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- list
- watch
- create
- delete
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- create
- delete
- apiGroups:
- apps
resources:
- daemonsets
verbs:
- get
- create
- patch
- delete
- apiGroups:
- ""
resources:
- services/proxy
verbs:
- get
- apiGroups:
- ""
resources:
- serviceaccounts
verbs:
- get
- create
- delete
- apiGroups:
- rbac.authorization.k8s.io
resources:
- roles
verbs:
- get
- create
- delete
- apiGroups:
- rbac.authorization.k8s.io
resources:
- rolebindings
verbs:
- get
- create
- delete
- apiGroups:
- apps
- extensions
resources:
- pods
verbs:
- get
- list
- watch
- apiGroups:
- apps
- extensions
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- ""
- apps
- extensions
resources:
- endpoints
verbs:
- get
- list
- watch
```

122
README.md
View File

@ -1,11 +1,22 @@
# 水 mizu
![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
# The API Traffic Viewer for Kubernetes
A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined.
![Simple UI](assets/mizu-ui.png)
## Features
- Simple and powerful CLI
- Real time view of all HTTP requests, REST and gRPC API calls
- No installation or code instrumentation
- Works completely on premises (on-prem)
## Download
Download `mizu` for your platform and operating system
Download Mizu for your platform and operating system
### Latest stable release
### Latest Stable Release
* for MacOS - Intel
```
@ -23,64 +34,45 @@ https://github.com/up9inc/mizu/releases/latest/download/mizu_linux_amd64 \
SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page.
### Development (unstable) build
### Development (unstable) Build
Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page.
## Prerequisites
1. Set `KUBECONFIG` environment variable to your kubernetes configuration. If this is not set, mizu assumes that configuration is at `${HOME}/.kube/config`
2. mizu needs following permissions on your kubernetes cluster to run
```
- apiGroups:
- ""
- apps
resources:
- pods
- services
verbs:
- list
- get
- create
- delete
- apiGroups:
- ""
- apps
resources:
- daemonsets
verbs:
- list
- get
- create
- patch
- delete
```
3. Optionally, for resolving traffic ip to kubernetes service name, mizu needs below permissions
```
- apiGroups:
- ""
- apps
- "rbac.authorization.k8s.io"
resources:
- clusterroles
- clusterrolebindings
- serviceaccounts
verbs:
- get
- create
- delete
```
1. Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config`
2. `mizu` assumes user running the command has permissions to create resources (such as pods, services, namespaces) on your Kubernetes cluster (no worries - `mizu` resources are cleaned up upon termination)
## How to run
For detailed list of k8s permissions see [PERMISSIONS](PERMISSIONS.md) document
1. Find pod you'd like to tap to in your Kubernetes cluster
2. Run `mizu tap PODNAME` or `mizu tap REGEX`
3. Open browser on `http://localhost:8899` as instructed ..
4. Watch the WebAPI traffic flowing ..
## How to Run
1. Find pods you'd like to tap to in your Kubernetes cluster
2. Run `mizu tap` or `mizu tap PODNAME`
3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI ..
4. Watch the API traffic flowing ..
5. Type ^C to stop
## Examples
Run `mizu help` for usage options
To tap all pods in current namespace -
```
$ kubectl get pods
NAME READY STATUS RESTARTS AGE
carts-66c77f5fbb-fq65r 2/2 Running 0 20m
catalogue-5f4cb7cf5-7zrmn 2/2 Running 0 20m
front-end-649fc5fd6-kqbtn 2/2 Running 0 20m
..
$ mizu tap
+carts-66c77f5fbb-fq65r
+catalogue-5f4cb7cf5-7zrmn
+front-end-649fc5fd6-kqbtn
Web interface is now available at http://localhost:8899
^C
```
To tap specific pod -
```
@ -111,3 +103,33 @@ To tap multiple pods using regex -
^C
```
## Configuration
Mizu can work with config file which should be stored in ${HOME}/.mizu/config.yaml (macOS: ~/.mizu/config.yaml) <br />
In case no config file found, defaults will be used. <br />
In case of partial configuration defined, all other fields will be used with defaults. <br />
You can always override the defaults or config file with CLI flags.
To get the default config params run `mizu config` <br />
To generate a new config file with default values use `mizu config -r`
Mizu has several undocumented flags which can be set by using --set flag (e.g., `mizu tap --set dump-logs=true`)
* **mizu-resources-namespace**: Type - String, See [Namespace-Restricted Mode](#namespace-restricted-mode)
* **telemetry**: Type - Boolean, Reports telemetry
* **dump-logs**: Type - Boolean, At the end of the execution it creates a zip file with logs (in .mizu folder)
* **kube-config-path**: Type - String, Setting the path to kube config (which isn't in standard path)
## Advanced Usage
### Namespace-Restricted Mode
Some users have permission to only manage resources in one particular namespace assigned to them.
By default `mizu tap` creates a new namespace `mizu` for all of its Kubernetes resources. In order to instead install
Mizu in an existing namespace, set the `mizu-resources-namespace` config option.
If `mizu-resources-namespace` is set to a value other than the default `mizu`, Mizu will operate in a
Namespace-Restricted mode. It will only tap pods in `mizu-resources-namespace`. This way Mizu only requires permissions
to the namespace set by `mizu-resources-namespace`. The user must set the tapped namespace to the same namespace by
using the `--namespace` flag or by setting `tap.namespaces` in the config file.
Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior.

6
api/README.md → agent/README.md
View File

@ -1,5 +1,5 @@
# mizu API server
API server for MIZU
# mizu agent
Agent for MIZU (API server and tapper)
Basic APIs:
* /fetch - retrieve traffic data
* /stats - retrieve statistics of collected data
@ -14,7 +14,7 @@ Basic APIs:
### Connecting
1. Start mizu using the cli with the debug image `mizu tap --mizu-image gcr.io/up9-docker-hub/mizu/debug:latest {tapped_pod_name}`
2. Forward the debug port using `kubectl port-forward -n default mizu-collector 2345:2345`
2. Forward the debug port using `kubectl port-forward -n default mizu-api-server 2345:2345`
3. Run the run/debug configuration you've created earlier in Intellij.
<small>Do note that dlv won't start the api until a debugger connects to it.</small>

11
api/go.mod → agent/go.mod
View File

@ -3,27 +3,28 @@ module mizuserver
go 1.16
require (
github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a
github.com/beevik/etree v1.1.0
github.com/djherbis/atime v1.0.0
github.com/fasthttp/websocket v1.4.3-beta.1 // indirect
github.com/fsnotify/fsnotify v1.4.9
github.com/gin-contrib/static v0.0.1
github.com/gin-gonic/gin v1.7.2
github.com/go-playground/locales v0.13.0
github.com/go-playground/universal-translator v0.17.0
github.com/go-playground/validator/v10 v10.5.0
github.com/gofiber/fiber/v2 v2.8.0
github.com/google/martian v2.1.0+incompatible
github.com/gorilla/websocket v1.4.2
github.com/leodido/go-urn v1.2.1 // indirect
github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
github.com/up9inc/mizu/shared v0.0.0
github.com/up9inc/mizu/tap v0.0.0
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
go.mongodb.org/mongo-driver v1.5.1
gorm.io/driver/sqlite v1.1.4
gorm.io/gorm v1.21.8
k8s.io/api v0.21.0
k8s.io/apimachinery v0.21.0
k8s.io/client-go v0.21.0
github.com/fsnotify/fsnotify v1.4.9
)
replace github.com/up9inc/mizu/shared v0.0.0 => ../shared

69
api/go.sum → agent/go.sum
View File

@ -41,15 +41,12 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc=
github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a h1:76llBleIE3fkdqaJFDzdirtiYhQPdIQem8H8r2iwA1Q=
github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a/go.mod h1:QvDfsDQDmGxUsvEeWabVZ5pp2FMXpOkwQV0L6SE6cp0=
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M=
github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@ -69,14 +66,18 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/fasthttp/websocket v1.4.2/go.mod h1:smsv/h4PBEBaU0XDTY5UwJTpZv69fQ0FfcLJr21mA6Y=
github.com/fasthttp/websocket v1.4.3-beta.1 h1:stc4P2aoxYKsdmbe1AJ5mAm73Fxc1NOgrZpPftvZIXQ=
github.com/fasthttp/websocket v1.4.3-beta.1/go.mod h1:JGrgLaT02bL9NuJkZbHN8mVV2tkCJZQh7yJ5/XCXO2g=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-contrib/static v0.0.1 h1:JVxuvHPuUfkoul12N7dtQw7KRn/pSMq7Ue1Va9Swm1U=
github.com/gin-contrib/static v0.0.1/go.mod h1:CSxeF+wep05e0kCOsqWdAWbSszmc31zTIbD8TvWl7Hs=
github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
github.com/gin-gonic/gin v1.7.2 h1:Tg03T9yM2xa8j6I3Z3oqLaQRSmKvxPd6g/2HJ6zICFA=
github.com/gin-gonic/gin v1.7.2/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@ -96,6 +97,8 @@ github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8c
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E=
github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
@ -124,12 +127,6 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe
github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
github.com/gofiber/fiber/v2 v2.1.3/go.mod h1:MMiSv1HrDkN8Pv7NeVDYK+T/lwXOEKAvPBbLvJPCEfA=
github.com/gofiber/fiber/v2 v2.7.1/go.mod h1:f8BRRIMjMdRyt2qmJ/0Sea3j3rwwfufPrh9WNBRiVZ0=
github.com/gofiber/fiber/v2 v2.8.0 h1:BdWvZmg/WY/Vjtjm38aXOp1Lks1BhuyS2b7lSWSPAzk=
github.com/gofiber/fiber/v2 v2.8.0/go.mod h1:Ah3IJikrKNRepl/HuVawppS25X7FWohwfCSRn7kJG28=
github.com/gofiber/websocket/v2 v2.0.3 h1:nqPGHB4LQhxKX5KJUjayOd2xiiENieS/dn6TPfCL8uk=
github.com/gofiber/websocket/v2 v2.0.3/go.mod h1:/OTEImCxORKE5unw0dWqJYovid6vZF+wB1W0aaMKs2M=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@ -190,7 +187,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
@ -202,6 +198,7 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@ -210,13 +207,7 @@ github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaR
github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4=
github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@ -226,13 +217,14 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ=
github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@ -255,6 +247,7 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU=
github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
@ -269,9 +262,6 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0=
github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY=
github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c h1:KKqhycXW1WVNkX7r4ekTV2gFkbhdyihlWD8c0/FiWmk=
github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8=
github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@ -283,28 +273,22 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
github.com/valyala/fasthttp v1.15.1/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
github.com/valyala/fasthttp v1.18.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A=
github.com/valyala/fasthttp v1.23.0 h1:0ufwSD9BhWa6f8HWdmdq4FHQ23peRo3Ng/Qs8m5NcFs=
github.com/valyala/fasthttp v1.23.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@ -377,11 +361,8 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210421230115-4e50805a0758 h1:aEpZnXcAmXkd6AvLb2OPt+EN1Zu/8Ne3pCqPjja5PXY=
golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@ -420,17 +401,15 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201210223839-7e3030f88018/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe h1:WdX7u8s3yOigWAhHEaDl8r9G+4XwFQEQFtBMYyN+kXQ=
golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

102
api/main.go → agent/main.go
View File

@ -4,38 +4,41 @@ import (
"encoding/json"
"flag"
"fmt"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/cors"
"github.com/gin-contrib/static"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
"mizuserver/pkg/api"
"mizuserver/pkg/middleware"
"mizuserver/pkg/models"
"mizuserver/pkg/routes"
"mizuserver/pkg/sensitiveDataFiltering"
"mizuserver/pkg/utils"
"net/http"
"os"
"os/signal"
"strings"
)
var shouldTap = flag.Bool("tap", false, "Run in tapper mode without API")
var aggregator = flag.Bool("aggregator", false, "Run in aggregator mode with API")
var standalone = flag.Bool("standalone", false, "Run in standalone tapper and API mode")
var aggregatorAddress = flag.String("aggregator-address", "", "Address of mizu collector for tapping")
var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API")
var apiServerMode = flag.Bool("api-server", false, "Run in API server mode with API")
var standaloneMode = flag.Bool("standalone", false, "Run in standalone tapper and API mode")
var apiServerAddress = flag.String("api-server-address", "", "Address of mizu API server")
var namespace = flag.String("namespace", "", "Resolve IPs if they belong to resources in this namespace (default is all)")
func main() {
flag.Parse()
hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
tapOpts := &tap.TapOpts{HostMode: hostMode}
if !*shouldTap && !*aggregator && !*standalone {
if !*tapperMode && !*apiServerMode && !*standaloneMode {
panic("One of the flags --tap, --api or --standalone must be provided")
}
if *standalone {
if *standaloneMode {
api.StartResolving(*namespace)
harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts)
filteredHarChannel := make(chan *tap.OutputChannelItem)
@ -44,9 +47,9 @@ func main() {
go api.StartReadingOutbound(outboundLinkOutputChannel)
hostApi(nil)
} else if *shouldTap {
if *aggregatorAddress == "" {
panic("Aggregator address must be provided with --aggregator-address when using --tap")
} else if *tapperMode {
if *apiServerAddress == "" {
panic("API server address must be provided with --api-server-address when using --tap")
}
tapTargets := getTapTargets()
@ -57,14 +60,16 @@ func main() {
harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts)
socketConnection, err := shared.ConnectToSocketServer(*aggregatorAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false)
socketConnection, err := shared.ConnectToSocketServer(*apiServerAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false)
if err != nil {
panic(fmt.Sprintf("Error connecting to socket server at %s %v", *aggregatorAddress, err))
panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
}
go pipeChannelToSocket(socketConnection, harOutputChannel)
go api.StartReadingOutbound(outboundLinkOutputChannel)
} else if *aggregator {
go pipeTapChannelToSocket(socketConnection, harOutputChannel)
go pipeOutboundLinksChannelToSocket(socketConnection, outboundLinkOutputChannel)
} else if *apiServerMode {
api.StartResolving(*namespace)
socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000)
filteredHarChannel := make(chan *tap.OutputChannelItem)
@ -82,31 +87,44 @@ func main() {
}
func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) {
app := fiber.New()
app := gin.Default()
app.Use(cors.New(cors.Config{
AllowOrigins: "*",
AllowMethods: "*",
AllowHeaders: "*",
}))
middleware.FiberMiddleware(app) // Register Fiber's middleware for app.
app.Static("/", "./site")
//Simple route to know server is running
app.Get("/echo", func(c *fiber.Ctx) error {
return c.SendString("Hello, World 👋!")
app.GET("/echo", func(c *gin.Context) {
c.String(http.StatusOK, "Here is Mizu agent")
})
eventHandlers := api.RoutesEventHandlers{
SocketHarOutChannel: socketHarOutputChannel,
}
routes.WebSocketRoutes(app, &eventHandlers)
app.Use(static.ServeRoot("/", "./site"))
app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
api.WebSocketRoutes(app, &eventHandlers)
routes.EntriesRoutes(app)
routes.MetadataRoutes(app)
routes.StatusRoutes(app)
routes.NotFoundRoute(app)
utils.StartServer(app)
}
func CORSMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT")
if c.Request.Method == "OPTIONS" {
c.AbortWithStatus(204)
return
}
c.Next()
}
}
func getTapTargets() []string {
nodeName := os.Getenv(shared.NodeNameEnvVar)
var tappedAddressesPerNodeDict map[string][]string
@ -143,7 +161,9 @@ func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *ta
continue
}
sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions)
if !filterOptions.DisableRedaction {
sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions)
}
outChannel <- message
}
@ -163,7 +183,7 @@ func isHealthCheckByUserAgent(message *tap.OutputChannelItem) bool {
return false
}
func pipeChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) {
func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) {
if connection == nil {
panic("Websocket connection is nil")
}
@ -186,3 +206,21 @@ func pipeChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *
}
}
}
func pipeOutboundLinksChannelToSocket(connection *websocket.Conn, outboundLinkChannel <-chan *tap.OutboundLink) {
for outboundLink := range outboundLinkChannel {
if outboundLink.SuggestedProtocol == tap.TLSProtocol {
marshaledData, err := models.CreateWebsocketOutboundLinkMessage(outboundLink)
if err != nil {
rlog.Infof("Error converting outbound link to json %s, (%v,%+v)", err, err, err)
continue
}
err = connection.WriteMessage(websocket.TextMessage, marshaledData)
if err != nil {
rlog.Infof("error sending outbound link message through socket server %s, (%v,%+v)", err, err, err)
continue
}
}
}
}

23
api/pkg/api/main.go → agent/pkg/api/main.go
View File

@ -5,10 +5,6 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/google/martian/har"
"github.com/romana/rlog"
"github.com/up9inc/mizu/tap"
"go.mongodb.org/mongo-driver/bson/primitive"
"mizuserver/pkg/holder"
"net/url"
"os"
@ -17,6 +13,11 @@ import (
"strings"
"time"
"github.com/google/martian/har"
"github.com/romana/rlog"
"github.com/up9inc/mizu/tap"
"go.mongodb.org/mongo-driver/bson/primitive"
"mizuserver/pkg/database"
"mizuserver/pkg/models"
"mizuserver/pkg/resolver"
@ -25,9 +26,9 @@ import (
var k8sResolver *resolver.Resolver
func init() {
func StartResolving(namespace string) {
errOut := make(chan error, 100)
res, err := resolver.NewFromInCluster(errOut)
res, err := resolver.NewFromInCluster(errOut, namespace)
if err != nil {
rlog.Infof("error creating k8s resolver %s", err)
return
@ -88,9 +89,9 @@ func startReadingFiles(workingDir string) {
for _, entry := range inputHar.Log.Entries {
time.Sleep(time.Millisecond * 250)
connectionInfo := &tap.ConnectionInfo{
ClientIP: fileInfo.Name(),
ClientIP: fileInfo.Name(),
ClientPort: "",
ServerIP: "",
ServerIP: "",
ServerPort: "",
IsOutgoing: false,
}
@ -118,7 +119,6 @@ func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) {
}
}
func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
entryBytes, _ := json.Marshal(entry)
serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL)
@ -167,8 +167,10 @@ func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil {
return
}
baseEntry.Rules = models.RunValidationRulesState(*entry, serviceName)
baseEntry.Latency = entry.Timings.Receive
baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry)
broadcastToBrowserClients(baseEntryBytes)
BroadcastToBrowserClients(baseEntryBytes)
}
func getServiceNameFromUrl(inputUrl string) (string, string) {
@ -196,6 +198,5 @@ func getEstimatedEntrySizeBytes(mizuEntry models.MizuEntry) int {
sizeBytes += 8 // SizeBytes bytes
sizeBytes += 1 // IsOutgoing bytes
return sizeBytes
}

118
agent/pkg/api/socket_routes.go Normal file
View File

@ -0,0 +1,118 @@
package api
import (
"errors"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared/debounce"
"net/http"
"sync"
"time"
)
type EventHandlers interface {
WebSocketConnect(socketId int, isTapper bool)
WebSocketDisconnect(socketId int, isTapper bool)
WebSocketMessage(socketId int, message []byte)
}
type SocketConnection struct {
connection *websocket.Conn
lock *sync.Mutex
eventHandlers EventHandlers
isTapper bool
}
var websocketUpgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
}
var websocketIdsLock = sync.Mutex{}
var connectedWebsockets map[int]*SocketConnection
var connectedWebsocketIdCounter = 0
func init() {
websocketUpgrader.CheckOrigin = func(r *http.Request) bool { return true } // like cors for web socket
connectedWebsockets = make(map[int]*SocketConnection, 0)
}
func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
app.GET("/ws", func(c *gin.Context) {
websocketHandler(c.Writer, c.Request, eventHandlers, false)
})
app.GET("/wsTapper", func(c *gin.Context) {
websocketHandler(c.Writer, c.Request, eventHandlers, true)
})
}
func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool) {
conn, err := websocketUpgrader.Upgrade(w, r, nil)
if err != nil {
rlog.Errorf("Failed to set websocket upgrade: %v", err)
return
}
websocketIdsLock.Lock()
connectedWebsocketIdCounter++
socketId := connectedWebsocketIdCounter
connectedWebsockets[socketId] = &SocketConnection{connection: conn, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
websocketIdsLock.Unlock()
defer func() {
socketCleanup(socketId, connectedWebsockets[socketId])
}()
eventHandlers.WebSocketConnect(socketId, isTapper)
for {
_, msg, err := conn.ReadMessage()
if err != nil {
rlog.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
break
}
eventHandlers.WebSocketMessage(socketId, msg)
}
}
func socketCleanup(socketId int, socketConnection *SocketConnection) {
err := socketConnection.connection.Close()
if err != nil {
rlog.Errorf("Error closing socket connection for socket id %d: %v\n", socketId, err)
}
websocketIdsLock.Lock()
connectedWebsockets[socketId] = nil
websocketIdsLock.Unlock()
socketConnection.eventHandlers.WebSocketDisconnect(socketId, socketConnection.isTapper)
}
var db = debounce.NewDebouncer(time.Second*5, func() {
rlog.Error("Successfully sent to socket")
})
func SendToSocket(socketId int, message []byte) error {
socketObj := connectedWebsockets[socketId]
if socketObj == nil {
return errors.New("Socket is disconnected")
}
var sent = false
time.AfterFunc(time.Second*5, func() {
if !sent {
rlog.Error("Socket timed out")
socketCleanup(socketId, socketObj)
}
})
socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
err := socketObj.connection.WriteMessage(1, message)
socketObj.lock.Unlock()
sent = true
return err
}

130
agent/pkg/api/socket_server_handlers.go Normal file
View File

@ -0,0 +1,130 @@
package api
import (
"encoding/json"
"fmt"
"mizuserver/pkg/models"
"mizuserver/pkg/providers"
"mizuserver/pkg/up9"
"sync"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
)
var browserClientSocketUUIDs = make([]int, 0)
var socketListLock = sync.Mutex{}
type RoutesEventHandlers struct {
EventHandlers
SocketHarOutChannel chan<- *tap.OutputChannelItem
}
func init() {
go up9.UpdateAnalyzeStatus(BroadcastToBrowserClients)
}
func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
if isTapper {
rlog.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
} else {
rlog.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
socketListLock.Lock()
browserClientSocketUUIDs = append(browserClientSocketUUIDs, socketId)
socketListLock.Unlock()
}
}
func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
if isTapper {
rlog.Infof("Websocket event - Tapper disconnected, socket ID: %d", socketId)
} else {
rlog.Infof("Websocket event - Browser socket disconnected, socket ID: %d", socketId)
socketListLock.Lock()
removeSocketUUIDFromBrowserSlice(socketId)
socketListLock.Unlock()
}
}
func BroadcastToBrowserClients(message []byte) {
for _, socketId := range browserClientSocketUUIDs {
go func(socketId int) {
err := SendToSocket(socketId, message)
if err != nil {
rlog.Errorf("error sending message to socket ID %d: %v", socketId, err)
}
}(socketId)
}
}
func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
var socketMessageBase shared.WebSocketMessageMetadata
err := json.Unmarshal(message, &socketMessageBase)
if err != nil {
rlog.Infof("Could not unmarshal websocket message %v\n", err)
} else {
switch socketMessageBase.MessageType {
case shared.WebSocketMessageTypeTappedEntry:
var tappedEntryMessage models.WebSocketTappedEntryMessage
err := json.Unmarshal(message, &tappedEntryMessage)
if err != nil {
rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
} else {
h.SocketHarOutChannel <- tappedEntryMessage.Data
}
case shared.WebSocketMessageTypeUpdateStatus:
var statusMessage shared.WebSocketStatusMessage
err := json.Unmarshal(message, &statusMessage)
if err != nil {
rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
} else {
providers.TapStatus.Pods = statusMessage.TappingStatus.Pods
BroadcastToBrowserClients(message)
}
case shared.WebsocketMessageTypeOutboundLink:
var outboundLinkMessage models.WebsocketOutboundLinkMessage
err := json.Unmarshal(message, &outboundLinkMessage)
if err != nil {
rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
} else {
handleTLSLink(outboundLinkMessage)
}
default:
rlog.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
}
}
}
func handleTLSLink(outboundLinkMessage models.WebsocketOutboundLinkMessage) {
resolvedName := k8sResolver.Resolve(outboundLinkMessage.Data.DstIP)
if resolvedName != "" {
outboundLinkMessage.Data.DstIP = resolvedName
} else if outboundLinkMessage.Data.SuggestedResolvedName != "" {
outboundLinkMessage.Data.DstIP = outboundLinkMessage.Data.SuggestedResolvedName
}
cacheKey := fmt.Sprintf("%s -> %s:%d", outboundLinkMessage.Data.Src, outboundLinkMessage.Data.DstIP, outboundLinkMessage.Data.DstPort)
_, isInCache := providers.RecentTLSLinks.Get(cacheKey)
if isInCache {
return
} else {
providers.RecentTLSLinks.SetDefault(cacheKey, outboundLinkMessage.Data)
}
marshaledMessage, err := json.Marshal(outboundLinkMessage)
if err != nil {
rlog.Errorf("Error marshaling outbound link message for broadcasting: %v", err)
} else {
rlog.Errorf("Broadcasting outboundlink message %s", string(marshaledMessage))
BroadcastToBrowserClients(marshaledMessage)
}
}
func removeSocketUUIDFromBrowserSlice(uuidToRemove int) {
newUUIDSlice := make([]int, 0, len(browserClientSocketUUIDs))
for _, uuid := range browserClientSocketUUIDs {
if uuid != uuidToRemove {
newUUIDSlice = append(newUUIDSlice, uuid)
}
}
browserClientSocketUUIDs = newUUIDSlice
}

108
api/pkg/controllers/entries_controller.go → agent/pkg/controllers/entries_controller.go
View File

@ -3,27 +3,30 @@ package controllers
import (
"encoding/json"
"fmt"
"github.com/gofiber/fiber/v2"
"github.com/google/martian/har"
"github.com/romana/rlog"
"mizuserver/pkg/database"
"mizuserver/pkg/models"
"mizuserver/pkg/providers"
"mizuserver/pkg/up9"
"mizuserver/pkg/utils"
"mizuserver/pkg/validation"
"net/http"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/google/martian/har"
"github.com/romana/rlog"
)
func GetEntries(c *fiber.Ctx) error {
func GetEntries(c *gin.Context) {
entriesFilter := &models.EntriesFilter{}
if err := c.QueryParser(entriesFilter); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
if err := c.BindQuery(entriesFilter); err != nil {
c.JSON(http.StatusBadRequest, err)
}
err := validation.Validate(entriesFilter)
if err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
c.JSON(http.StatusBadRequest, err)
}
order := database.OperatorToOrderMapping[entriesFilter.Operator]
@ -50,18 +53,18 @@ func GetEntries(c *fiber.Ctx) error {
baseEntries = append(baseEntries, harEntry)
}
return c.Status(fiber.StatusOK).JSON(baseEntries)
c.JSON(http.StatusOK, baseEntries)
}
func GetHARs(c *fiber.Ctx) error {
entriesFilter := &models.HarFetchRequestBody{}
func GetHARs(c *gin.Context) {
entriesFilter := &models.HarFetchRequestQuery{}
order := database.OrderDesc
if err := c.QueryParser(entriesFilter); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
if err := c.BindQuery(entriesFilter); err != nil {
c.JSON(http.StatusBadRequest, err)
}
err := validation.Validate(entriesFilter)
if err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
c.JSON(http.StatusBadRequest, err)
}
var timestampFrom, timestampTo int64
@ -137,40 +140,45 @@ func GetHARs(c *fiber.Ctx) error {
retObj[k] = bytesData
}
buffer := utils.ZipData(retObj)
return c.Status(fiber.StatusOK).SendStream(buffer)
c.Data(http.StatusOK, "application/octet-stream", buffer.Bytes())
}
func UploadEntries(c *fiber.Ctx) error {
func UploadEntries(c *gin.Context) {
rlog.Infof("Upload entries - started\n")
uploadRequestBody := &models.UploadEntriesRequestBody{}
if err := c.QueryParser(uploadRequestBody); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
uploadParams := &models.UploadEntriesRequestQuery{}
if err := c.BindQuery(uploadParams); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
if err := validation.Validate(uploadRequestBody); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
if err := validation.Validate(uploadParams); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
if up9.GetAnalyzeInfo().IsAnalyzing {
return c.Status(fiber.StatusBadRequest).SendString("Cannot analyze, mizu is already analyzing")
c.String(http.StatusBadRequest, "Cannot analyze, mizu is already analyzing")
return
}
rlog.Infof("Upload entries - creating token. dest %s\n", uploadRequestBody.Dest)
token, err := up9.CreateAnonymousToken(uploadRequestBody.Dest)
rlog.Infof("Upload entries - creating token. dest %s\n", uploadParams.Dest)
token, err := up9.CreateAnonymousToken(uploadParams.Dest)
if err != nil {
return c.Status(fiber.StatusServiceUnavailable).SendString("Can't get token")
c.String(http.StatusServiceUnavailable, "Cannot analyze, mizu is already analyzing")
return
}
rlog.Infof("Upload entries - uploading. token: %s model: %s\n", token.Token, token.Model)
go up9.UploadEntriesImpl(token.Token, token.Model, uploadRequestBody.Dest, uploadRequestBody.SleepIntervalSec)
return c.Status(fiber.StatusOK).SendString("OK")
go up9.UploadEntriesImpl(token.Token, token.Model, uploadParams.Dest, uploadParams.SleepIntervalSec)
c.String(http.StatusOK, "OK")
}
func GetFullEntries(c *fiber.Ctx) error {
entriesFilter := &models.HarFetchRequestBody{}
if err := c.QueryParser(entriesFilter); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
func GetFullEntries(c *gin.Context) {
entriesFilter := &models.HarFetchRequestQuery{}
if err := c.BindQuery(entriesFilter); err != nil {
c.JSON(http.StatusBadRequest, err)
}
err := validation.Validate(entriesFilter)
if err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
c.JSON(http.StatusBadRequest, err)
}
var timestampFrom, timestampTo int64
@ -195,38 +203,44 @@ func GetFullEntries(c *fiber.Ctx) error {
}
result = append(result, harEntry)
}
return c.Status(fiber.StatusOK).JSON(result)
c.JSON(http.StatusOK, result)
}
func GetEntry(c *fiber.Ctx) error {
func GetEntry(c *gin.Context) {
var entryData models.MizuEntry
database.GetEntriesTable().
Where(map[string]string{"entryId": c.Params("entryId")}).
Where(map[string]string{"entryId": c.Param("entryId")}).
First(&entryData)
fullEntry := models.FullEntryDetails{}
if err := models.GetEntry(&entryData, &fullEntry); err != nil {
return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
c.JSON(http.StatusInternalServerError, map[string]interface{}{
"error": true,
"msg": "Can't get entry details",
})
}
return c.Status(fiber.StatusOK).JSON(fullEntry)
fullEntryWithPolicy := models.FullEntryWithPolicy{}
if err := models.GetEntry(&entryData, &fullEntryWithPolicy); err != nil {
c.JSON(http.StatusInternalServerError, map[string]interface{}{
"error": true,
"msg": "Can't get entry details",
})
}
c.JSON(http.StatusOK, fullEntryWithPolicy)
}
func DeleteAllEntries(c *fiber.Ctx) error {
func DeleteAllEntries(c *gin.Context) {
database.GetEntriesTable().
Where("1 = 1").
Delete(&models.MizuEntry{})
return c.Status(fiber.StatusOK).JSON(fiber.Map{
c.JSON(http.StatusOK, map[string]string{
"msg": "Success",
})
}
func GetGeneralStats(c *fiber.Ctx) error {
func GetGeneralStats(c *gin.Context) {
sqlQuery := "SELECT count(*) as count, min(timestamp) as min, max(timestamp) as max from mizu_entries"
var result struct {
Count int
@ -234,5 +248,17 @@ func GetGeneralStats(c *fiber.Ctx) error {
Max int
}
database.GetEntriesTable().Raw(sqlQuery).Scan(&result)
return c.Status(fiber.StatusOK).JSON(&result)
c.JSON(http.StatusOK, result)
}
func GetTappingStatus(c *gin.Context) {
c.JSON(http.StatusOK, providers.TapStatus)
}
func AnalyzeInformation(c *gin.Context) {
c.JSON(http.StatusOK, up9.GetAnalyzeInfo())
}
func GetRecentTLSLinks(c *gin.Context) {
c.JSON(http.StatusOK, providers.GetAllRecentTLSAddresses())
}

7
api/pkg/controllers/metadata_controller.go → agent/pkg/controllers/metadata_controller.go
View File

@ -1,12 +1,13 @@
package controllers
import (
"github.com/gofiber/fiber/v2"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/shared"
"mizuserver/pkg/version"
"net/http"
)
func GetVersion(c *fiber.Ctx) error {
func GetVersion(c *gin.Context) {
resp := shared.VersionResponse{SemVer: version.SemVer}
return c.Status(fiber.StatusOK).JSON(resp)
c.JSON(http.StatusOK, resp)
}

12
agent/pkg/controllers/resolving_controller.go Normal file
View File

@ -0,0 +1,12 @@
package controllers
import (
"github.com/gin-gonic/gin"
"mizuserver/pkg/holder"
"net/http"
)
func GetCurrentResolvingInformation(c *gin.Context) {
c.JSON(http.StatusOK, holder.GetResolver().GetMap())
}

32
agent/pkg/controllers/status_controller.go Normal file
View File

@ -0,0 +1,32 @@
package controllers
import (
"encoding/json"
"github.com/gin-gonic/gin"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared"
"mizuserver/pkg/api"
"mizuserver/pkg/providers"
"mizuserver/pkg/validation"
"net/http"
)
func PostTappedPods(c *gin.Context) {
tapStatus := &shared.TapStatus{}
if err := c.Bind(tapStatus); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
if err := validation.Validate(tapStatus); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
rlog.Infof("[Status] POST request: %d tapped pods", len(tapStatus.Pods))
providers.TapStatus.Pods = tapStatus.Pods
message := shared.CreateWebSocketStatusMessage(*tapStatus)
if jsonBytes, err := json.Marshal(message); err != nil {
rlog.Errorf("Could not Marshal message %v\n", err)
} else {
api.BroadcastToBrowserClients(jsonBytes)
}
}

0
api/pkg/database/main.go → agent/pkg/database/main.go
View File

18
api/pkg/database/size_enforcer.go → agent/pkg/database/size_enforcer.go
View File

@ -1,8 +1,8 @@
package database
import (
"fmt"
"github.com/fsnotify/fsnotify"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/debounce"
"github.com/up9inc/mizu/shared/units"
@ -47,7 +47,7 @@ func StartEnforcingDatabaseSize() {
if !ok {
return // closed channel
}
fmt.Printf("filesystem watcher encountered error:%v\n", err)
rlog.Errorf("filesystem watcher encountered error:%v", err)
}
}
}()
@ -62,7 +62,7 @@ func getMaxEntriesDBByteSize() (int64, error) {
maxEntriesDBByteSize := defaultMaxDatabaseSizeBytes
var err error
maxEntriesDBSizeByteSEnvVarValue := os.Getenv(shared.MaxEntriesDBSizeByteSEnvVar)
maxEntriesDBSizeByteSEnvVarValue := os.Getenv(shared.MaxEntriesDBSizeBytesEnvVar)
if maxEntriesDBSizeByteSEnvVarValue != "" {
maxEntriesDBByteSize, err = strconv.ParseInt(maxEntriesDBSizeByteSEnvVarValue, 10, 64)
}
@ -72,7 +72,7 @@ func getMaxEntriesDBByteSize() (int64, error) {
func checkFileSize(maxSizeBytes int64) {
fileStat, err := os.Stat(DBPath)
if err != nil {
fmt.Printf("Error checking %s file size: %v\n", DBPath, err)
rlog.Errorf("Error checking %s file size: %v", DBPath, err)
} else {
if fileStat.Size() > maxSizeBytes {
pruneOldEntries(fileStat.Size())
@ -83,13 +83,13 @@ func checkFileSize(maxSizeBytes int64) {
func pruneOldEntries(currentFileSize int64) {
// sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here
IsDBLocked = true
defer func() {IsDBLocked = false}()
defer func() { IsDBLocked = false }()
amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune)
rows, err := GetEntriesTable().Limit(10000).Order("id").Rows()
if err != nil {
fmt.Printf("Error getting 10000 first db rows: %v\n", err)
rlog.Errorf("Error getting 10000 first db rows: %v", err)
return
}
@ -102,7 +102,7 @@ func pruneOldEntries(currentFileSize int64) {
var entry models.MizuEntry
err = DB.ScanRows(rows, &entry)
if err != nil {
fmt.Printf("Error scanning db row: %v\n", err)
rlog.Errorf("Error scanning db row: %v", err)
continue
}
@ -114,8 +114,8 @@ func pruneOldEntries(currentFileSize int64) {
GetEntriesTable().Where(entryIdsToRemove).Delete(models.MizuEntry{})
// VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this
DB.Exec("VACUUM")
fmt.Printf("Removed %d rows and cleared %s\n", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved))
rlog.Errorf("Removed %d rows and cleared %s", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved))
} else {
fmt.Println("Found no rows to remove when pruning")
rlog.Error("Found no rows to remove when pruning")
}
}

0
api/pkg/holder/main.go → agent/pkg/holder/main.go
View File

103
api/pkg/models/models.go → agent/pkg/models/models.go
View File

@ -2,11 +2,14 @@ package models
import (
"encoding/json"
"mizuserver/pkg/rules"
"mizuserver/pkg/utils"
"time"
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
"mizuserver/pkg/utils"
"time"
)
type DataUnmarshaler interface {
@ -33,19 +36,33 @@ type MizuEntry struct {
ResolvedSource string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"`
ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
IsOutgoing bool `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"`
EstimatedSizeBytes int `json:"-" gorm:"column:estimatedSizeBytes"`
EstimatedSizeBytes int `json:"-" gorm:"column:estimatedSizeBytes"`
}
type BaseEntryDetails struct {
Id string `json:"id,omitempty"`
Url string `json:"url,omitempty"`
RequestSenderIp string `json:"requestSenderIp,omitempty"`
Service string `json:"service,omitempty"`
Path string `json:"path,omitempty"`
StatusCode int `json:"statusCode,omitempty"`
Method string `json:"method,omitempty"`
Timestamp int64 `json:"timestamp,omitempty"`
IsOutgoing bool `json:"isOutgoing,omitempty"`
Id string `json:"id,omitempty"`
Url string `json:"url,omitempty"`
RequestSenderIp string `json:"requestSenderIp,omitempty"`
Service string `json:"service,omitempty"`
Path string `json:"path,omitempty"`
StatusCode int `json:"statusCode,omitempty"`
Method string `json:"method,omitempty"`
Timestamp int64 `json:"timestamp,omitempty"`
IsOutgoing bool `json:"isOutgoing,omitempty"`
Latency int64 `json:"latency,omitempty"`
Rules ApplicableRules `json:"rules,omitempty"`
}
type ApplicableRules struct {
Latency int64 `json:"latency,omitempty"`
Status bool `json:"status,omitempty"`
}
func NewApplicableRules(status bool, latency int64) ApplicableRules {
ar := ApplicableRules{}
ar.Status = status
ar.Latency = latency
return ar
}
type FullEntryDetails struct {
@ -101,25 +118,20 @@ func (fedex *FullEntryDetailsExtra) UnmarshalData(entry *MizuEntry) error {
return nil
}
type EntryData struct {
Entry string `json:"entry,omitempty"`
ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
}
type EntriesFilter struct {
Limit int `query:"limit" validate:"required,min=1,max=200"`
Operator string `query:"operator" validate:"required,oneof='lt' 'gt'"`
Timestamp int64 `query:"timestamp" validate:"required,min=1"`
Limit int `form:"limit" validate:"required,min=1,max=200"`
Operator string `form:"operator" validate:"required,oneof='lt' 'gt'"`
Timestamp int64 `form:"timestamp" validate:"required,min=1"`
}
type UploadEntriesRequestBody struct {
Dest string `query:"dest"`
SleepIntervalSec int `query:"interval"`
type UploadEntriesRequestQuery struct {
Dest string `form:"dest"`
SleepIntervalSec int `form:"interval"`
}
type HarFetchRequestBody struct {
From int64 `query:"from"`
To int64 `query:"to"`
type HarFetchRequestQuery struct {
From int64 `form:"from"`
To int64 `form:"to"`
}
type WebSocketEntryMessage struct {
@ -132,6 +144,11 @@ type WebSocketTappedEntryMessage struct {
Data *tap.OutputChannelItem
}
type WebsocketOutboundLinkMessage struct {
*shared.WebSocketMessageMetadata
Data *tap.OutboundLink
}
func CreateBaseEntryWebSocketMessage(base *BaseEntryDetails) ([]byte, error) {
message := &WebSocketEntryMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
@ -152,6 +169,16 @@ func CreateWebsocketTappedEntryMessage(base *tap.OutputChannelItem) ([]byte, err
return json.Marshal(message)
}
func CreateWebsocketOutboundLinkMessage(base *tap.OutboundLink) ([]byte, error) {
message := &WebsocketOutboundLinkMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
MessageType: shared.WebsocketMessageTypeOutboundLink,
},
Data: base,
}
return json.Marshal(message)
}
// ExtendedHAR is the top level object of a HAR log.
type ExtendedHAR struct {
Log *ExtendedLog `json:"log"`
@ -171,3 +198,27 @@ type ExtendedCreator struct {
*har.Creator
Source *string `json:"_source"`
}
type FullEntryWithPolicy struct {
RulesMatched []rules.RulesMatched `json:"rulesMatched,omitempty"`
Entry har.Entry `json:"entry"`
Service string `json:"service"`
}
func (fewp *FullEntryWithPolicy) UnmarshalData(entry *MizuEntry) error {
if err := json.Unmarshal([]byte(entry.Entry), &fewp.Entry); err != nil {
return err
}
_, resultPolicyToSend := rules.MatchRequestPolicy(fewp.Entry, entry.Service)
fewp.RulesMatched = resultPolicyToSend
fewp.Service = entry.Service
return nil
}
func RunValidationRulesState(harEntry har.Entry, service string) ApplicableRules {
numberOfRules, resultPolicyToSend := rules.MatchRequestPolicy(harEntry, service)
statusPolicyToSend, latency := rules.PassedValidationRules(resultPolicyToSend, numberOfRules)
ar := NewApplicableRules(statusPolicyToSend, latency)
return ar
}

28
agent/pkg/providers/status_provider.go Normal file
View File

@ -0,0 +1,28 @@
package providers
import (
"github.com/patrickmn/go-cache"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
"time"
)
const tlsLinkRetainmentTime = time.Minute * 15
var (
TapStatus shared.TapStatus
RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
)
func GetAllRecentTLSAddresses() []string {
recentTLSLinks := make([]string, 0)
for _, outboundLinkItem := range RecentTLSLinks.Items() {
outboundLink, castOk := outboundLinkItem.Object.(*tap.OutboundLink)
if castOk {
recentTLSLinks = append(recentTLSLinks, outboundLink.DstIP)
}
}
return recentTLSLinks
}

8
api/pkg/resolver/README.md → agent/pkg/resolver/README.md
View File

@ -32,7 +32,7 @@ Now you will be able to import `github.com/up9inc/mizu/resolver` in any `.go` fi
errOut := make(chan error, 100)
k8sResolver, err := resolver.NewFromOutOfCluster("", errOut)
if err != nil {
fmt.Printf("error creating k8s resolver %s", err)
rlog.Errorf("error creating k8s resolver %s", err)
}
ctx, cancel := context.WithCancel(context.Background())
@ -40,15 +40,15 @@ k8sResolver.Start(ctx)
resolvedName := k8sResolver.Resolve("10.107.251.91") // will always return `nil` in real scenarios as the internal map takes a moment to populate after `Start` is called
if resolvedName != nil {
fmt.Printf("resolved 10.107.251.91=%s", *resolvedName)
rlog.Errorf("resolved 10.107.251.91=%s", *resolvedName)
} else {
fmt.Printf("Could not find a resolved name for 10.107.251.91")
rlog.Error("Could not find a resolved name for 10.107.251.91")
}
for {
select {
case err := <- errOut:
fmt.Printf("name resolving error %s", err)
rlog.Errorf("name resolving error %s", err)
}
}
```

0
api/pkg/resolver/go.sum → agent/pkg/resolver/go.sum
View File

23
agent/pkg/resolver/loader.go Normal file
View File

@ -0,0 +1,23 @@
package resolver
import (
cmap "github.com/orcaman/concurrent-map"
"k8s.io/client-go/kubernetes"
_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
restclient "k8s.io/client-go/rest"
)
func NewFromInCluster(errOut chan error, namesapce string) (*Resolver, error) {
config, err := restclient.InClusterConfig()
if err != nil {
return nil, err
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, err
}
return &Resolver{clientConfig: config, clientSet: clientset, nameMap: cmap.New(), serviceMap: cmap.New(), errOut: errOut, namespace: namesapce}, nil
}

119
api/pkg/resolver/resolver.go → agent/pkg/resolver/resolver.go
View File

@ -7,28 +7,32 @@ import (
"github.com/romana/rlog"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"github.com/orcaman/concurrent-map"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
)
const (
kubClientNullString = "None"
)
type Resolver struct {
clientConfig *restclient.Config
clientSet *kubernetes.Clientset
nameMap map[string]string
serviceMap map[string]string
isStarted bool
errOut chan error
clientConfig *restclient.Config
clientSet *kubernetes.Clientset
nameMap cmap.ConcurrentMap
serviceMap cmap.ConcurrentMap
isStarted bool
errOut chan error
namespace string
}
func (resolver *Resolver) Start(ctx context.Context) {
if !resolver.isStarted {
resolver.isStarted = true
go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchServices)
go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchEndpoints)
go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchPods)
@ -36,97 +40,97 @@ func (resolver *Resolver) Start(ctx context.Context) {
}
func (resolver *Resolver) Resolve(name string) string {
resolvedName, isFound := resolver.nameMap[name]
resolvedName, isFound := resolver.nameMap.Get(name)
if !isFound {
return ""
}
return resolvedName
return resolvedName.(string)
}
func (resolver *Resolver) GetMap() map[string]string {
func (resolver *Resolver) GetMap() cmap.ConcurrentMap {
return resolver.nameMap
}
func (resolver *Resolver) CheckIsServiceIP(address string) bool {
_, isFound := resolver.serviceMap[address]
_, isFound := resolver.serviceMap.Get(address)
return isFound
}
func (resolver *Resolver) watchPods(ctx context.Context) error {
// empty namespace makes the client watch all namespaces
watcher, err := resolver.clientSet.CoreV1().Pods("").Watch(ctx, metav1.ListOptions{Watch: true})
watcher, err := resolver.clientSet.CoreV1().Pods(resolver.namespace).Watch(ctx, metav1.ListOptions{Watch: true})
if err != nil {
return err
}
for {
select {
case event := <- watcher.ResultChan():
if event.Object == nil {
return errors.New("error in kubectl pod watch")
}
if event.Type == watch.Deleted {
pod := event.Object.(*corev1.Pod)
resolver.saveResolvedName(pod.Status.PodIP, "", event.Type)
}
case <- ctx.Done():
watcher.Stop()
return nil
case event := <-watcher.ResultChan():
if event.Object == nil {
return errors.New("error in kubectl pod watch")
}
if event.Type == watch.Deleted {
pod := event.Object.(*corev1.Pod)
resolver.saveResolvedName(pod.Status.PodIP, "", event.Type)
}
case <-ctx.Done():
watcher.Stop()
return nil
}
}
}
func (resolver *Resolver) watchEndpoints(ctx context.Context) error {
// empty namespace makes the client watch all namespaces
watcher, err := resolver.clientSet.CoreV1().Endpoints("").Watch(ctx, metav1.ListOptions{Watch: true})
watcher, err := resolver.clientSet.CoreV1().Endpoints(resolver.namespace).Watch(ctx, metav1.ListOptions{Watch: true})
if err != nil {
return err
}
for {
select {
case event := <- watcher.ResultChan():
if event.Object == nil {
return errors.New("error in kubectl endpoint watch")
}
endpoint := event.Object.(*corev1.Endpoints)
serviceHostname := fmt.Sprintf("%s.%s", endpoint.Name, endpoint.Namespace)
if endpoint.Subsets != nil {
for _, subset := range endpoint.Subsets {
var ports []int32
if subset.Ports != nil {
for _, portMapping := range subset.Ports {
if portMapping.Port > 0 {
ports = append(ports, portMapping.Port)
}
case event := <-watcher.ResultChan():
if event.Object == nil {
return errors.New("error in kubectl endpoint watch")
}
endpoint := event.Object.(*corev1.Endpoints)
serviceHostname := fmt.Sprintf("%s.%s", endpoint.Name, endpoint.Namespace)
if endpoint.Subsets != nil {
for _, subset := range endpoint.Subsets {
var ports []int32
if subset.Ports != nil {
for _, portMapping := range subset.Ports {
if portMapping.Port > 0 {
ports = append(ports, portMapping.Port)
}
}
if subset.Addresses != nil {
for _, address := range subset.Addresses {
resolver.saveResolvedName(address.IP, serviceHostname, event.Type)
for _, port := range ports {
ipWithPort := fmt.Sprintf("%s:%d", address.IP, port)
resolver.saveResolvedName(ipWithPort, serviceHostname, event.Type)
}
}
}
}
if subset.Addresses != nil {
for _, address := range subset.Addresses {
resolver.saveResolvedName(address.IP, serviceHostname, event.Type)
for _, port := range ports {
ipWithPort := fmt.Sprintf("%s:%d", address.IP, port)
resolver.saveResolvedName(ipWithPort, serviceHostname, event.Type)
}
}
}
}
case <- ctx.Done():
watcher.Stop()
return nil
}
case <-ctx.Done():
watcher.Stop()
return nil
}
}
}
func (resolver *Resolver) watchServices(ctx context.Context) error {
// empty namespace makes the client watch all namespaces
watcher, err := resolver.clientSet.CoreV1().Services("").Watch(ctx, metav1.ListOptions{Watch: true})
watcher, err := resolver.clientSet.CoreV1().Services(resolver.namespace).Watch(ctx, metav1.ListOptions{Watch: true})
if err != nil {
return err
}
for {
select {
case event := <- watcher.ResultChan():
case event := <-watcher.ResultChan():
if event.Object == nil {
return errors.New("error in kubectl service watch")
}
@ -142,7 +146,7 @@ func (resolver *Resolver) watchServices(ctx context.Context) error {
resolver.saveResolvedName(ingress.IP, serviceHostname, event.Type)
}
}
case <- ctx.Done():
case <-ctx.Done():
watcher.Stop()
return nil
}
@ -151,19 +155,19 @@ func (resolver *Resolver) watchServices(ctx context.Context) error {
func (resolver *Resolver) saveResolvedName(key string, resolved string, eventType watch.EventType) {
if eventType == watch.Deleted {
delete(resolver.nameMap, key)
resolver.nameMap.Remove(key)
rlog.Infof("setting %s=nil\n", key)
} else {
resolver.nameMap[key] = resolved
resolver.nameMap.Set(key, resolved)
rlog.Infof("setting %s=%s\n", key, resolved)
}
}
func (resolver *Resolver) saveServiceIP(key string, resolved string, eventType watch.EventType) {
if eventType == watch.Deleted {
delete(resolver.serviceMap, key)
resolver.serviceMap.Remove(key)
} else {
resolver.serviceMap[key] = resolved
resolver.serviceMap.Set(key, resolved)
}
}
@ -186,4 +190,3 @@ func (resolver *Resolver) infiniteErrorHandleRetryFunc(ctx context.Context, fun
}
}
}

26
agent/pkg/routes/entries_routes.go Normal file
View File

@ -0,0 +1,26 @@
package routes
import (
"github.com/gin-gonic/gin"
"mizuserver/pkg/controllers"
)
// EntriesRoutes defines the group of har entries routes.
func EntriesRoutes(ginApp *gin.Engine) {
routeGroup := ginApp.Group("/api")
routeGroup.GET("/entries", controllers.GetEntries) // get entries (base/thin entries)
routeGroup.GET("/entries/:entryId", controllers.GetEntry) // get single (full) entry
routeGroup.GET("/exportEntries", controllers.GetFullEntries)
routeGroup.GET("/uploadEntries", controllers.UploadEntries)
routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation)
routeGroup.GET("/har", controllers.GetHARs)
routeGroup.GET("/resetDB", controllers.DeleteAllEntries) // get single (full) entry
routeGroup.GET("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB
routeGroup.GET("/tapStatus", controllers.GetTappingStatus) // get tapping status
routeGroup.GET("/analyzeStatus", controllers.AnalyzeInformation)
routeGroup.GET("/recentTLSLinks", controllers.GetRecentTLSLinks)
}

13
agent/pkg/routes/metadata_routes.go Normal file
View File

@ -0,0 +1,13 @@
package routes
import (
"github.com/gin-gonic/gin"
"mizuserver/pkg/controllers"
)
// MetadataRoutes defines the group of metadata routes.
func MetadataRoutes(app *gin.Engine) {
routeGroup := app.Group("/metadata")
routeGroup.GET("/version", controllers.GetVersion)
}

18
agent/pkg/routes/not_found_route.go Normal file
View File

@ -0,0 +1,18 @@
package routes
import (
"github.com/gin-gonic/gin"
"net/http"
)
// NotFoundRoute defines the 404 Error route.
func NotFoundRoute(app *gin.Engine) {
app.Use(
func(c *gin.Context) {
c.JSON(http.StatusNotFound, map[string]interface{}{
"error": true,
"msg": "sorry, endpoint is not found",
})
},
)
}

12
agent/pkg/routes/status_routes.go Normal file
View File

@ -0,0 +1,12 @@
package routes
import (
"github.com/gin-gonic/gin"
"mizuserver/pkg/controllers"
)
func StatusRoutes(ginApp *gin.Engine) {
routeGroup := ginApp.Group("/status")
routeGroup.POST("/tappedPods", controllers.PostTappedPods)
}

110
agent/pkg/rules/models.go Normal file
View File

@ -0,0 +1,110 @@
package rules
import (
"encoding/json"
"fmt"
"reflect"
"regexp"
"strings"
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared"
jsonpath "github.com/yalp/jsonpath"
)
type RulesMatched struct {
Matched bool `json:"matched"`
Rule shared.RulePolicy `json:"rule"`
}
func appendRulesMatched(rulesMatched []RulesMatched, matched bool, rule shared.RulePolicy) []RulesMatched {
return append(rulesMatched, RulesMatched{Matched: matched, Rule: rule})
}
func ValidatePath(URLFromRule string, URL string) bool {
if URLFromRule != "" {
matchPath, err := regexp.MatchString(URLFromRule, URL)
if err != nil || !matchPath {
return false
}
}
return true
}
func ValidateService(serviceFromRule string, service string) bool {
if serviceFromRule != "" {
matchService, err := regexp.MatchString(serviceFromRule, service)
if err != nil || !matchService {
return false
}
}
return true
}
func MatchRequestPolicy(harEntry har.Entry, service string) (int, []RulesMatched) {
enforcePolicy, _ := shared.DecodeEnforcePolicy(fmt.Sprintf("%s/%s", shared.RulePolicyPath, shared.RulePolicyFileName))
var resultPolicyToSend []RulesMatched
for _, rule := range enforcePolicy.Rules {
if !ValidatePath(rule.Path, harEntry.Request.URL) || !ValidateService(rule.Service, service) {
continue
}
if rule.Type == "json" {
var bodyJsonMap interface{}
if err := json.Unmarshal(harEntry.Response.Content.Text, &bodyJsonMap); err != nil {
continue
}
out, err := jsonpath.Read(bodyJsonMap, rule.Key)
if err != nil || out == nil {
continue
}
var matchValue bool
if reflect.TypeOf(out).Kind() == reflect.String {
matchValue, err = regexp.MatchString(rule.Value, out.(string))
if err != nil {
continue
}
} else {
val := fmt.Sprint(out)
matchValue, err = regexp.MatchString(rule.Value, val)
if err != nil {
continue
}
}
resultPolicyToSend = appendRulesMatched(resultPolicyToSend, matchValue, rule)
} else if rule.Type == "header" {
for j := range harEntry.Response.Headers {
matchKey, err := regexp.MatchString(rule.Key, harEntry.Response.Headers[j].Name)
if err != nil {
continue
}
if matchKey {
matchValue, err := regexp.MatchString(rule.Value, harEntry.Response.Headers[j].Value)
if err != nil {
continue
}
resultPolicyToSend = appendRulesMatched(resultPolicyToSend, matchValue, rule)
}
}
} else {
resultPolicyToSend = appendRulesMatched(resultPolicyToSend, true, rule)
}
}
return len(enforcePolicy.Rules), resultPolicyToSend
}
func PassedValidationRules(rulesMatched []RulesMatched, numberOfRules int) (bool, int64) {
if len(rulesMatched) == 0 {
return false, 0
}
for _, rule := range rulesMatched {
if rule.Matched == false {
return false, -1
}
}
for _, rule := range rulesMatched {
if strings.ToLower(rule.Rule.Type) == "latency" {
return true, rule.Rule.Latency
}
}
return true, -1
}

0
api/pkg/sensitiveDataFiltering/consts.go → agent/pkg/sensitiveDataFiltering/consts.go
View File

0
api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go → agent/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go
View File

0
api/pkg/up9/main.go → agent/pkg/up9/main.go
View File

0
api/pkg/utils/pathUtils.go → agent/pkg/utils/pathUtils.go
View File

0
api/pkg/utils/randomString.go → agent/pkg/utils/randomString.go
View File

9
api/pkg/utils/truncating_logger.go → agent/pkg/utils/truncating_logger.go
View File

@ -3,6 +3,7 @@ package utils
import (
"context"
"fmt"
"github.com/romana/rlog"
"gorm.io/gorm/logger"
"gorm.io/gorm/utils"
"time"
@ -10,7 +11,7 @@ import (
// TruncatingLogger implements the gorm logger.Interface interface. Its purpose is to act as gorm's logger while truncating logs to a max of 50 characters to minimise the performance impact
type TruncatingLogger struct {
LogLevel logger.LogLevel
LogLevel logger.LogLevel
SlowThreshold time.Duration
}
@ -23,21 +24,21 @@ func (truncatingLogger *TruncatingLogger) Info(_ context.Context, message string
if truncatingLogger.LogLevel < logger.Info {
return
}
fmt.Printf("gorm info: %.150s\n", message)
rlog.Errorf("gorm info: %.150s", message)
}
func (truncatingLogger *TruncatingLogger) Warn(_ context.Context, message string, __ ...interface{}) {
if truncatingLogger.LogLevel < logger.Warn {
return
}
fmt.Printf("gorm warning: %.150s\n", message)
rlog.Errorf("gorm warning: %.150s", message)
}
func (truncatingLogger *TruncatingLogger) Error(_ context.Context, message string, __ ...interface{}) {
if truncatingLogger.LogLevel < logger.Error {
return
}
fmt.Printf("gorm error: %.150s\n", message)
rlog.Errorf("gorm error: %.150s", message)
}
func (truncatingLogger *TruncatingLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) {

18
api/pkg/utils/utils.go → agent/pkg/utils/utils.go
View File

@ -1,32 +1,42 @@
package utils
import (
"github.com/gofiber/fiber/v2"
"context"
"github.com/gin-gonic/gin"
"github.com/romana/rlog"
"log"
"net/http"
"net/url"
"os"
"os/signal"
"reflect"
"syscall"
"time"
)
// StartServer starts the server with a graceful shutdown
func StartServer(app *fiber.App) {
func StartServer(app *gin.Engine) {
signals := make(chan os.Signal, 2)
signal.Notify(signals,
os.Interrupt, // this catch ctrl + c
syscall.SIGTSTP, // this catch ctrl + z
)
srv := &http.Server{
Addr: ":8080",
Handler: app,
}
go func() {
_ = <-signals
rlog.Infof("Shutting down...")
_ = app.Shutdown()
ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
_ = srv.Shutdown(ctx)
os.Exit(0)
}()
// Run server.
if err := app.Listen(":8899"); err != nil {
if err := app.Run(":8899"); err != nil {
log.Printf("Oops... Server is not running! Reason: %v", err)
}
}

0
api/pkg/utils/zip.go → agent/pkg/utils/zip.go
View File

0
api/pkg/validation/validation.go → agent/pkg/validation/validation.go
View File

0
api/pkg/version/consts.go → agent/pkg/version/consts.go
View File

0
api/start.sh → agent/start.sh
View File

99
api/pkg/api/socket_server_handlers.go
View File

@ -1,99 +0,0 @@
package api
import (
"encoding/json"
"github.com/antoniodipinto/ikisocket"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
"mizuserver/pkg/controllers"
"mizuserver/pkg/models"
"mizuserver/pkg/routes"
"mizuserver/pkg/up9"
)
var browserClientSocketUUIDs = make([]string, 0)
type RoutesEventHandlers struct {
routes.EventHandlers
SocketHarOutChannel chan<- *tap.OutputChannelItem
}
func init() {
go up9.UpdateAnalyzeStatus(broadcastToBrowserClients)
}
func (h *RoutesEventHandlers) WebSocketConnect(ep *ikisocket.EventPayload) {
if ep.Kws.GetAttribute("is_tapper") == true {
rlog.Infof("Websocket Connection event - Tapper connected: %s", ep.SocketUUID)
} else {
rlog.Infof("Websocket Connection event - Browser socket connected: %s", ep.SocketUUID)
browserClientSocketUUIDs = append(browserClientSocketUUIDs, ep.SocketUUID)
}
}
func (h *RoutesEventHandlers) WebSocketDisconnect(ep *ikisocket.EventPayload) {
if ep.Kws.GetAttribute("is_tapper") == true {
rlog.Infof("Disconnection event - Tapper connected: %s", ep.SocketUUID)
} else {
rlog.Infof("Disconnection event - Browser socket connected: %s", ep.SocketUUID)
removeSocketUUIDFromBrowserSlice(ep.SocketUUID)
}
}
func broadcastToBrowserClients(message []byte) {
ikisocket.EmitToList(browserClientSocketUUIDs, message)
}
func (h *RoutesEventHandlers) WebSocketClose(ep *ikisocket.EventPayload) {
if ep.Kws.GetAttribute("is_tapper") == true {
rlog.Infof("Websocket Close event - Tapper connected: %s", ep.SocketUUID)
} else {
rlog.Infof("Websocket Close event - Browser socket connected: %s", ep.SocketUUID)
removeSocketUUIDFromBrowserSlice(ep.SocketUUID)
}
}
func (h *RoutesEventHandlers) WebSocketError(ep *ikisocket.EventPayload) {
rlog.Infof("Socket error - Socket uuid : %s %v", ep.SocketUUID, ep.Error)
}
func (h *RoutesEventHandlers) WebSocketMessage(ep *ikisocket.EventPayload) {
var socketMessageBase shared.WebSocketMessageMetadata
err := json.Unmarshal(ep.Data, &socketMessageBase)
if err != nil {
rlog.Infof("Could not unmarshal websocket message %v\n", err)
} else {
switch socketMessageBase.MessageType {
case shared.WebSocketMessageTypeTappedEntry:
var tappedEntryMessage models.WebSocketTappedEntryMessage
err := json.Unmarshal(ep.Data, &tappedEntryMessage)
if err != nil {
rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
} else {
h.SocketHarOutChannel <- tappedEntryMessage.Data
}
case shared.WebSocketMessageTypeUpdateStatus:
var statusMessage shared.WebSocketStatusMessage
err := json.Unmarshal(ep.Data, &statusMessage)
if err != nil {
rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
} else {
controllers.TapStatus = statusMessage.TappingStatus
broadcastToBrowserClients(ep.Data)
}
default:
rlog.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
}
}
}
func removeSocketUUIDFromBrowserSlice(uuidToRemove string) {
newUUIDSlice := make([]string, 0, len(browserClientSocketUUIDs))
for _, uuid := range browserClientSocketUUIDs {
if uuid != uuidToRemove {
newUUIDSlice = append(newUUIDSlice, uuid)
}
}
browserClientSocketUUIDs = newUUIDSlice
}

11
api/pkg/controllers/resolving_controller.go
View File

@ -1,11 +0,0 @@
package controllers
import (
"github.com/gofiber/fiber/v2"
"mizuserver/pkg/holder"
)
func GetCurrentResolvingInformation(c *fiber.Ctx) error {
return c.Status(fiber.StatusOK).JSON(holder.GetResolver().GetMap())
}

17
api/pkg/controllers/status_controller.go
View File

@ -1,17 +0,0 @@
package controllers
import (
"github.com/gofiber/fiber/v2"
"github.com/up9inc/mizu/shared"
"mizuserver/pkg/up9"
)
var TapStatus shared.TapStatus
func GetTappingStatus(c *fiber.Ctx) error {
return c.Status(fiber.StatusOK).JSON(TapStatus)
}
func AnalyzeInformation(c *fiber.Ctx) error {
return c.Status(fiber.StatusOK).JSON(up9.GetAnalyzeInfo())
}

18
api/pkg/middleware/fiber_middleware.go
View File

@ -1,18 +0,0 @@
package middleware
import (
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/cors"
"github.com/gofiber/fiber/v2/middleware/logger"
)
// FiberMiddleware provide Fiber's built-in middlewares.
// See: https://docs.gofiber.io/api/middleware
func FiberMiddleware(a *fiber.App) {
a.Use(
// Add CORS to each route.
cors.New(),
// Add simple logger.
logger.New(),
)
}

67
api/pkg/resolver/loader.go
View File

@ -1,67 +0,0 @@
package resolver
import (
"k8s.io/client-go/kubernetes"
_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
"path/filepath"
"os"
)
func NewFromInCluster(errOut chan error) (*Resolver, error) {
config, err := restclient.InClusterConfig()
if err != nil {
return nil, err
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, err
}
return &Resolver{clientConfig: config, clientSet: clientset, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}, nil
}
func NewFromOutOfCluster(kubeConfigPath string, errOut chan error) (*Resolver, error) {
if kubeConfigPath == "" {
env := os.Getenv("KUBECONFIG")
if env != "" {
kubeConfigPath = env
} else {
home := homedir.HomeDir()
kubeConfigPath = filepath.Join(home, ".kube", "config")
}
}
configPathList := filepath.SplitList(kubeConfigPath)
configLoadingRules := &clientcmd.ClientConfigLoadingRules{}
if len(configPathList) <= 1 {
configLoadingRules.ExplicitPath = kubeConfigPath
} else {
configLoadingRules.Precedence = configPathList
}
contextName := ""
clientConfigLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
configLoadingRules,
&clientcmd.ConfigOverrides{
CurrentContext: contextName,
},
)
clientConfig, err := clientConfigLoader.ClientConfig()
if err != nil {
return nil, err
}
clientset, err := kubernetes.NewForConfig(clientConfig)
if err != nil {
return nil, err
}
return &Resolver{clientConfig: clientConfig, clientSet: clientset, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}, nil
}
func NewFromExisting(clientConfig *restclient.Config, clientSet *kubernetes.Clientset, errOut chan error) *Resolver {
return &Resolver{clientConfig: clientConfig, clientSet: clientSet, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}
}

25
api/pkg/routes/entries_routes.go
View File

@ -1,25 +0,0 @@
package routes
import (
"github.com/gofiber/fiber/v2"
"mizuserver/pkg/controllers"
)
// EntriesRoutes defines the group of har entries routes.
func EntriesRoutes(fiberApp *fiber.App) {
routeGroup := fiberApp.Group("/api")
routeGroup.Get("/entries", controllers.GetEntries) // get entries (base/thin entries)
routeGroup.Get("/entries/:entryId", controllers.GetEntry) // get single (full) entry
routeGroup.Get("/exportEntries", controllers.GetFullEntries)
routeGroup.Get("/uploadEntries", controllers.UploadEntries)
routeGroup.Get("/resolving", controllers.GetCurrentResolvingInformation)
routeGroup.Get("/har", controllers.GetHARs)
routeGroup.Get("/resetDB", controllers.DeleteAllEntries) // get single (full) entry
routeGroup.Get("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB
routeGroup.Get("/tapStatus", controllers.GetTappingStatus) // get tapping status
routeGroup.Get("/analyzeStatus", controllers.AnalyzeInformation)
}

13
api/pkg/routes/metadata_routes.go
View File

@ -1,13 +0,0 @@
package routes
import (
"github.com/gofiber/fiber/v2"
"mizuserver/pkg/controllers"
)
// MetadataRoutes defines the group of metadata routes.
func MetadataRoutes(fiberApp *fiber.App) {
routeGroup := fiberApp.Group("/metadata")
routeGroup.Get("/version", controllers.GetVersion)
}

15
api/pkg/routes/not_found_route.go
View File

@ -1,15 +0,0 @@
package routes
import "github.com/gofiber/fiber/v2"
// NotFoundRoute defines the 404 Error route.
func NotFoundRoute(fiberApp *fiber.App) {
fiberApp.Use(
func(c *fiber.Ctx) error {
return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
"error": true,
"msg": "sorry, endpoint is not found",
})
},
)
}

31
api/pkg/routes/socket_routes.go
View File

@ -1,31 +0,0 @@
package routes
import (
"github.com/antoniodipinto/ikisocket"
"github.com/gofiber/fiber/v2"
)
type EventHandlers interface {
WebSocketConnect(ep *ikisocket.EventPayload)
WebSocketDisconnect(ep *ikisocket.EventPayload)
WebSocketClose(ep *ikisocket.EventPayload)
WebSocketError(ep *ikisocket.EventPayload)
WebSocketMessage(ep *ikisocket.EventPayload)
}
func WebSocketRoutes(app *fiber.App, eventHandlers EventHandlers) {
app.Get("/ws", ikisocket.New(func(kws *ikisocket.Websocket) {
kws.SetAttribute("is_tapper", false)
}))
app.Get("/wsTapper", ikisocket.New(func(kws *ikisocket.Websocket) {
// Tapper clients are handled differently, they don't need to receive new message broadcasts.
kws.SetAttribute("is_tapper", true)
}))
ikisocket.On(ikisocket.EventMessage, eventHandlers.WebSocketMessage)
ikisocket.On(ikisocket.EventConnect, eventHandlers.WebSocketConnect)
ikisocket.On(ikisocket.EventDisconnect, eventHandlers.WebSocketDisconnect)
ikisocket.On(ikisocket.EventClose, eventHandlers.WebSocketClose) // This event is called when the server disconnects the user actively with .Close() method
ikisocket.On(ikisocket.EventError, eventHandlers.WebSocketError) // On error event
}

BIN
assets/mizu-example.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 811 KiB

24
assets/mizu-logo.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 44 KiB

BIN
assets/mizu-ui.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 491 KiB

16
build-push-featurebranch.sh
View File

@ -5,7 +5,9 @@ SERVER_NAME=mizu
GCP_PROJECT=up9-docker-hub
REPOSITORY=gcr.io/$GCP_PROJECT
GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
DOCKER_TAGGED_BUILD=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH:latest
SEM_VER=${SEM_VER=0.0.0}
DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
then
@ -13,8 +15,12 @@ then
exit 1
fi
echo "building $DOCKER_TAGGED_BUILD"
docker build -t "$DOCKER_TAGGED_BUILD" --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
echo "building ${DOCKER_TAGGED_BUILDS[@]}"
DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
echo pushing to "$REPOSITORY"
docker push "$DOCKER_TAGGED_BUILD"
for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
do
echo pushing "$DOCKER_TAG"
docker push "$DOCKER_TAG"
done

1
cli/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
bin

7
cli/Makefile
View File

@ -3,6 +3,7 @@ COMMIT_HASH=$(shell git rev-parse HEAD)
GIT_BRANCH=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
GIT_VERSION=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
BUILD_TIMESTAMP=$(shell date +%s)
export SEM_VER?=0.0.0
.PHONY: help
.DEFAULT_GOAL := help
@ -13,7 +14,7 @@ help: ## This help.
install:
go install mizu.go
build: ## build mizu CLI binary (select platform via GOOS / GOARCH env variables)
build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables).
go build -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
-X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
-X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
@ -21,7 +22,7 @@ build: ## build mizu CLI binary (select platform via GOOS / GOARCH env variables
-o bin/mizu_$(SUFFIX) mizu.go
(cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256)
build-all: ## build for all supported platforms
build-all: ## Build for all supported platforms.
@echo "Compiling for every OS and Platform"
@mkdir -p bin && echo "SHA256 checksums available for compiled binaries \n\nRun \`shasum -a 256 -c mizu_OS_ARCH.sha256\` to verify\n\n" > bin/README.md
@$(MAKE) build GOOS=darwin GOARCH=amd64
@ -35,6 +36,6 @@ build-all: ## build for all supported platforms
@echo "---------"
@find ./bin -ls
clean: ## clean all build artifacts
clean: ## Clean all build artifacts.
go clean
rm -rf ./bin/*

26
cli/README.md
View File

@ -1,26 +0,0 @@
# mizu CLI
## Usage
`./mizu {pod_name_regex}`
### Optional Flags
| flag | default | purpose |
|----------------------|------------------|--------------------------------------------------------------------------------------------------------------|
| `--no-gui` | `false` | Don't host the web interface (not applicable at the moment) |
| `--gui-port` | `8899` | local port that web interface will be forwarded to |
| `--namespace` | | use namespace different than the one found in kubeconfig |
| `--kubeconfig` | | Path to custom kubeconfig file |
There are some extra flags defined in code that will show up in `./mizu --help`, these are non functional stubs for now
## Installation
Make sure your go version is at least 1.11
1. cd to `mizu/cli`
2. Run `go mod download` (may take a moment)
3. Run `go build mizu.go`
Alternatively, you can build+run directly using `go run mizu.go {pod_name_regex}`
## Known issues
* mid-flight port forwarding failures are not detected and no indication will be shown when this occurs

40
cli/cmd/config.go Normal file
View File

@ -0,0 +1,40 @@
package cmd
import (
"fmt"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"io/ioutil"
)
var regenerateFile bool
var configCmd = &cobra.Command{
Use: "config",
Short: "Generate config with default values",
RunE: func(cmd *cobra.Command, args []string) error {
template, err := mizu.GetConfigWithDefaults()
if err != nil {
mizu.Log.Errorf("Failed generating config with defaults %v", err)
return nil
}
if regenerateFile {
data := []byte(template)
if err := ioutil.WriteFile(mizu.GetConfigFilePath(), data, 0644); err != nil {
mizu.Log.Errorf("Failed writing config %v", err)
return nil
}
mizu.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, mizu.GetConfigFilePath())))
} else {
mizu.Log.Debugf("Writing template config.\n%v", template)
fmt.Printf("%v", template)
}
return nil
},
}
func init() {
rootCmd.AddCommand(configCmd)
configCmd.Flags().BoolVarP(&regenerateFile, "regenerate", "r", false, fmt.Sprintf("Regenerate the config file with default values %s", mizu.GetConfigFilePath()))
}

27
cli/cmd/fetch.go
View File

@ -1,29 +1,23 @@
package cmd
import (
"github.com/creasty/defaults"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/mizu/configStructs"
)
type MizuFetchOptions struct {
FromTimestamp int64
ToTimestamp int64
Directory string
MizuPort uint16
}
var mizuFetchOptions = MizuFetchOptions{}
var fetchCmd = &cobra.Command{
Use: "fetch",
Short: "Download recorded traffic to files",
RunE: func(cmd *cobra.Command, args []string) error {
if isCompatible, err := mizu.CheckVersionCompatibility(mizuFetchOptions.MizuPort); err != nil {
go mizu.ReportRun("fetch", mizu.Config.Fetch)
if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.Fetch.GuiPort); err != nil {
return err
} else if !isCompatible {
return nil
}
RunMizuFetch(&mizuFetchOptions)
RunMizuFetch()
return nil
},
}
@ -31,8 +25,11 @@ var fetchCmd = &cobra.Command{
func init() {
rootCmd.AddCommand(fetchCmd)
fetchCmd.Flags().StringVarP(&mizuFetchOptions.Directory, "directory", "d", ".", "Provide a custom directory for fetched entries")
fetchCmd.Flags().Int64Var(&mizuFetchOptions.FromTimestamp, "from", 0, "Custom start timestamp for fetched entries")
fetchCmd.Flags().Int64Var(&mizuFetchOptions.ToTimestamp, "to", 0, "Custom end timestamp fetched entries")
fetchCmd.Flags().Uint16VarP(&mizuFetchOptions.MizuPort, "port", "p", 8899, "Custom port for mizu")
defaultFetchConfig := configStructs.FetchConfig{}
defaults.Set(&defaultFetchConfig)
fetchCmd.Flags().StringP(configStructs.DirectoryFetchName, "d", defaultFetchConfig.Directory, "Provide a custom directory for fetched entries")
fetchCmd.Flags().Int(configStructs.FromTimestampFetchName, defaultFetchConfig.FromTimestamp, "Custom start timestamp for fetched entries")
fetchCmd.Flags().Int(configStructs.ToTimestampFetchName, defaultFetchConfig.ToTimestamp, "Custom end timestamp fetched entries")
fetchCmd.Flags().Uint16P(configStructs.GuiPortFetchName, "p", defaultFetchConfig.GuiPort, "Provide a custom port for the web interface webserver")
}

13
cli/cmd/fetchRunner.go
View File

@ -5,6 +5,7 @@ import (
"bytes"
"fmt"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"io"
"io/ioutil"
"log"
@ -14,9 +15,9 @@ import (
"strings"
)
func RunMizuFetch(fetch *MizuFetchOptions) {
mizuProxiedUrl := kubernetes.GetMizuCollectorProxiedHostAndPath(fetch.MizuPort)
resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, fetch.FromTimestamp, fetch.ToTimestamp))
func RunMizuFetch() {
mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Fetch.GuiPort)
resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, mizu.Config.Fetch.FromTimestamp, mizu.Config.Fetch.ToTimestamp))
if err != nil {
log.Fatal(err)
}
@ -32,8 +33,8 @@ func RunMizuFetch(fetch *MizuFetchOptions) {
if err != nil {
log.Fatal(err)
}
_ = Unzip(zipReader, fetch.Directory)
_ = Unzip(zipReader, mizu.Config.Fetch.Directory)
}
func Unzip(reader *zip.Reader, dest string) error {
@ -63,7 +64,7 @@ func Unzip(reader *zip.Reader, dest string) error {
_ = os.MkdirAll(path, f.Mode())
} else {
_ = os.MkdirAll(filepath.Dir(path), f.Mode())
fmt.Print("writing HAR file [ ", path, " ] .. ")
mizu.Log.Infof("writing HAR file [ %v ]", path)
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
if err != nil {
return err
@ -72,7 +73,7 @@ func Unzip(reader *zip.Reader, dest string) error {
if err := f.Close(); err != nil {
panic(err)
}
fmt.Println(" done")
mizu.Log.Info(" done")
}()
_, err = io.Copy(f, rc)

46
cli/cmd/logs.go Normal file
View File

@ -0,0 +1,46 @@
package cmd
import (
"context"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/fsUtils"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"os"
"path"
)
var filePath string
var logsCmd = &cobra.Command{
Use: "logs",
Short: "Create a zip file with logs for Github issue or troubleshoot",
RunE: func(cmd *cobra.Command, args []string) error {
kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.View.KubeConfigPath)
if err != nil {
return nil
}
ctx, _ := context.WithCancel(context.Background())
if filePath == "" {
pwd, err := os.Getwd()
if err != nil {
mizu.Log.Errorf("Failed to get PWD, %v (try using `mizu logs -f <full path dest zip file>)`", err)
return nil
}
filePath = path.Join(pwd, "mizu_logs.zip")
}
mizu.Log.Debugf("Using file path %s", filePath)
if err := fsUtils.DumpLogs(kubernetesProvider, ctx, filePath); err != nil {
mizu.Log.Errorf("Failed dump logs %v", err)
}
return nil
},
}
func init() {
rootCmd.AddCommand(logsCmd)
logsCmd.Flags().StringVarP(&filePath, "file", "f", "", "Path for zip file (default current <pwd>\\mizu_logs.zip)")
}

18
cli/cmd/root.go
View File

@ -1,7 +1,10 @@
package cmd
import (
"fmt"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/fsUtils"
"github.com/up9inc/mizu/cli/mizu"
)
var rootCmd = &cobra.Command{
@ -9,6 +12,21 @@ var rootCmd = &cobra.Command{
Short: "A web traffic viewer for kubernetes",
Long: `A web traffic viewer for kubernetes
Further info is available at https://github.com/up9inc/mizu`,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := fsUtils.EnsureDir(mizu.GetMizuFolderPath()); err != nil {
mizu.Log.Errorf("Failed to use mizu folder, %v", err)
}
mizu.InitLogger()
if err := mizu.InitConfig(cmd); err != nil {
mizu.Log.Fatal(err)
}
return nil
},
}
func init() {
rootCmd.PersistentFlags().StringSlice(mizu.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", mizu.SetCommandName))
}
// Execute adds all child commands to the root command and sets flags appropriately.

106
cli/cmd/tap.go
View File

@ -2,41 +2,17 @@ package cmd
import (
"errors"
"fmt"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared/units"
"os"
"regexp"
"strings"
"github.com/creasty/defaults"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/errormessage"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/mizu/configStructs"
"github.com/up9inc/mizu/cli/uiUtils"
)
type MizuTapOptions struct {
GuiPort uint16
Namespace string
AllNamespaces bool
Analysis bool
AnalysisDestination string
KubeConfigPath string
MizuImage string
PlainTextFilterRegexes []string
TapOutgoing bool
HideHealthChecks bool
MaxEntriesDBSizeBytes int64
SleepIntervalSec uint16
}
var mizuTapOptions = &MizuTapOptions{}
var direction string
var humanMaxEntriesDBSize string
var regex *regexp.Regexp
const maxEntriesDBSizeFlagName = "max-entries-db-size"
const analysisMessageToConfirm = `NOTE: running mizu with --analysis flag will upload recorded traffic
to UP9 cloud for further analysis and enriched presentation options.
`
const analysisMessageToConfirm = `NOTE: running mizu with --analysis flag will upload recorded traffic for further analysis and enriched presentation options.`
var tapCmd = &cobra.Command{
Use: "tap [POD REGEX]",
@ -44,45 +20,35 @@ var tapCmd = &cobra.Command{
Long: `Record the ingoing traffic of a kubernetes pod.
Supported protocols are HTTP and gRPC.`,
RunE: func(cmd *cobra.Command, args []string) error {
RunMizuTap(regex, mizuTapOptions)
go mizu.ReportRun("tap", mizu.Config.Tap)
RunMizuTap()
return nil
},
PreRunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return errors.New("POD REGEX argument is required")
if len(args) == 1 {
mizu.Config.Tap.PodRegexStr = args[0]
} else if len(args) > 1 {
return errors.New("unexpected number of arguments")
}
var compileErr error
regex, compileErr = regexp.Compile(args[0])
if compileErr != nil {
return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], compileErr))
if err := mizu.Config.Validate(); err != nil {
return errormessage.FormatError(err)
}
var parseHumanDataSizeErr error
mizuTapOptions.MaxEntriesDBSizeBytes, parseHumanDataSizeErr = units.HumanReadableToBytes(humanMaxEntriesDBSize)
if parseHumanDataSizeErr != nil {
return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize))
}
fmt.Printf("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes))
directionLowerCase := strings.ToLower(direction)
if directionLowerCase == "any" {
mizuTapOptions.TapOutgoing = true
} else if directionLowerCase == "in" {
mizuTapOptions.TapOutgoing = false
} else {
return errors.New(fmt.Sprintf("%s is not a valid value for flag --direction. Acceptable values are in/any.", direction))
if err := mizu.Config.Tap.Validate(); err != nil {
return errormessage.FormatError(err)
}
if mizuTapOptions.Analysis {
fmt.Printf(analysisMessageToConfirm)
if !uiUtils.AskForConfirmation("Would you like to proceed [y/n]: ") {
fmt.Println("You can always run mizu without analysis, aborting")
mizu.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", mizu.Config.Tap.HumanMaxEntriesDBSize)
if mizu.Config.Tap.Analysis {
mizu.Log.Infof(analysisMessageToConfirm)
if !uiUtils.AskForConfirmation("Would you like to proceed [Y/n]: ") {
mizu.Log.Infof("You can always run mizu without analysis, aborting")
os.Exit(0)
}
}
return nil
},
}
@ -90,16 +56,18 @@ Supported protocols are HTTP and gRPC.`,
func init() {
rootCmd.AddCommand(tapCmd)
tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector")
tapCmd.Flags().BoolVar(&mizuTapOptions.Analysis, "analysis", false, "Uploads traffic to UP9 for further analysis (Beta)")
tapCmd.Flags().StringVar(&mizuTapOptions.AnalysisDestination, "dest", "up9.app", "Destination environment")
tapCmd.Flags().Uint16VarP(&mizuTapOptions.SleepIntervalSec, "upload-interval", "", 10, "Interval in seconds for uploading data to UP9")
tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces")
tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file")
tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", mizu.Branch, mizu.SemVer), "Custom image for mizu collector")
tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "regex-masking", "r", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies")
tapCmd.Flags().StringVarP(&direction, "direction", "", "in", "Record traffic that goes in this direction (relative to the tapped pod): in/any")
tapCmd.Flags().BoolVar(&mizuTapOptions.HideHealthChecks, "hide-healthchecks", false, "hides requests with kube-probe or prometheus user-agent headers")
tapCmd.Flags().StringVarP(&humanMaxEntriesDBSize, maxEntriesDBSizeFlagName, "", "200MB", "override the default max entries db size of 200mb")
defaultTapConfig := configStructs.TapConfig{}
defaults.Set(&defaultTapConfig)
tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
tapCmd.Flags().StringArrayP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector")
tapCmd.Flags().Bool(configStructs.AnalysisTapName, defaultTapConfig.Analysis, "Uploads traffic to UP9 for further analysis (Beta)")
tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
tapCmd.Flags().StringArrayP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
tapCmd.Flags().Bool(configStructs.HideHealthChecksTapName, defaultTapConfig.HideHealthChecks, "Hides requests with kube-probe or prometheus user-agent headers")
tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any")
tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them")
tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file with policy rules")
}

583
cli/cmd/tapRunner.go
View File

@ -1,153 +1,240 @@
package cmd
import (
"bytes"
"context"
"encoding/json"
"fmt"
"github.com/romana/rlog"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/debounce"
core "k8s.io/api/core/v1"
"log"
"github.com/up9inc/mizu/cli/fsUtils"
"github.com/up9inc/mizu/cli/goUtils"
"github.com/up9inc/mizu/cli/mizu/configStructs"
"net/http"
"net/url"
"os"
"os/signal"
"path"
"regexp"
"strings"
"syscall"
"time"
"github.com/up9inc/mizu/cli/errormessage"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/debounce"
yaml "gopkg.in/yaml.v3"
core "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/wait"
)
var mizuServiceAccountExists bool
var aggregatorService *core.Service
const (
cleanupTimeout = time.Minute
updateTappersDelay = 5 * time.Second
)
var currentlyTappedPods []core.Pod
type tapState struct {
apiServerService *core.Service
currentlyTappedPods []core.Pod
mizuServiceAccountExists bool
doNotRemoveConfigMap bool
}
func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
mizuApiFilteringOptions, err := getMizuApiFilteringOptions(tappingOptions)
var state tapState
func RunMizuTap() {
mizuApiFilteringOptions, err := getMizuApiFilteringOptions()
if err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error parsing regex-masking: %v", errormessage.FormatError(err)))
return
}
var mizuValidationRules string
if mizu.Config.Tap.EnforcePolicyFile != "" {
mizuValidationRules, err = readValidationRules(mizu.Config.Tap.EnforcePolicyFile)
if err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading policy file: %v", errormessage.FormatError(err)))
return
}
}
kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.KubeConfigPath)
if err != nil {
mizu.Log.Error(err)
return
}
kubernetesProvider := kubernetes.NewProvider(tappingOptions.KubeConfigPath)
defer cleanUpMizuResources(kubernetesProvider)
ctx, cancel := context.WithCancel(context.Background())
defer cancel() // cancel will be called when this function exits
targetNamespace := getNamespace(tappingOptions, kubernetesProvider)
if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegexQuery, targetNamespace); err != nil {
return
} else {
currentlyTappedPods = matchingPods
}
targetNamespaces := getNamespaces(kubernetesProvider)
var namespacesStr string
if targetNamespace != mizu.K8sAllNamespaces {
namespacesStr = fmt.Sprintf("namespace \"%s\"", targetNamespace)
if targetNamespaces[0] != mizu.K8sAllNamespaces {
namespacesStr = fmt.Sprintf("namespaces \"%s\"", strings.Join(targetNamespaces, "\", \""))
} else {
namespacesStr = "all namespaces"
}
fmt.Printf("Tapping pods in %s\n", namespacesStr)
mizu.CheckNewerVersion()
mizu.Log.Infof("Tapping pods in %s", namespacesStr)
if len(currentlyTappedPods) == 0 {
if err, _ := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespaces); err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error getting pods by regex: %v", errormessage.FormatError(err)))
return
}
if len(state.currentlyTappedPods) == 0 {
var suggestionStr string
if targetNamespace != mizu.K8sAllNamespaces {
suggestionStr = "\nSelect a different namespace with -n or tap all namespaces with -A"
if targetNamespaces[0] != mizu.K8sAllNamespaces {
suggestionStr = ". Select a different namespace with -n or tap all namespaces with -A"
}
fmt.Printf("Did not find any pods matching the regex argument%s\n", suggestionStr)
mizu.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any pods matching the regex argument%s", suggestionStr))
}
nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods)
if err != nil {
if mizu.Config.Tap.DryRun {
return
}
if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions, mizuApiFilteringOptions); err != nil {
nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
defer cleanUpMizuResources(kubernetesProvider)
if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions, mizuValidationRules); err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
return
}
go portForwardApiPod(ctx, kubernetesProvider, cancel, tappingOptions) // TODO convert this to job for built in pod ttl or have the running app handle this
go watchPodsForTapping(ctx, kubernetesProvider, cancel, podRegexQuery, tappingOptions)
go syncApiStatus(ctx, cancel, tappingOptions)
go goUtils.HandleExcWrapper(createProxyToApiServerPod, ctx, kubernetesProvider, cancel)
go goUtils.HandleExcWrapper(watchPodsForTapping, ctx, kubernetesProvider, targetNamespaces, cancel)
//block until exit signal or error
waitForFinish(ctx, cancel)
}
func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
if err := createMizuAggregator(ctx, kubernetesProvider, tappingOptions, mizuApiFilteringOptions); err != nil {
func readValidationRules(file string) (string, error) {
rules, err := shared.DecodeEnforcePolicy(file)
if err != nil {
return "", err
}
newContent, _ := yaml.Marshal(&rules)
return string(newContent), nil
}
func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, mizuApiFilteringOptions *shared.TrafficFilteringOptions, mizuValidationRules string) error {
if !mizu.Config.IsNsRestrictedMode() {
if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
return err
}
}
if err := createMizuApiServer(ctx, kubernetesProvider, mizuApiFilteringOptions); err != nil {
return err
}
if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
return err
}
if err := createMizuConfigmap(ctx, kubernetesProvider, mizuValidationRules); err != nil {
mizu.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err)))
state.doNotRemoveConfigMap = true
} else if mizuValidationRules == "" {
state.doNotRemoveConfigMap = true
}
return nil
}
func createMizuAggregator(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, data string) error {
err := kubernetesProvider.CreateConfigMap(ctx, mizu.Config.MizuResourcesNamespace, mizu.ConfigMapName, data)
return err
}
func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Provider) error {
_, err := kubernetesProvider.CreateNamespace(ctx, mizu.Config.MizuResourcesNamespace)
return err
}
func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
var err error
mizuServiceAccountExists = createRBACIfNecessary(ctx, kubernetesProvider)
_, err = kubernetesProvider.CreateMizuAggregatorPod(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.MizuImage, mizuServiceAccountExists, mizuApiFilteringOptions, tappingOptions.MaxEntriesDBSizeBytes)
state.mizuServiceAccountExists, err = createRBACIfNecessary(ctx, kubernetesProvider)
if err != nil {
fmt.Printf("Error creating mizu collector pod: %v\n", err)
return err
mizu.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err)))
}
aggregatorService, err = kubernetesProvider.CreateService(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, mizu.AggregatorPodName)
var serviceAccountName string
if state.mizuServiceAccountExists {
serviceAccountName = mizu.ServiceAccountName
} else {
serviceAccountName = ""
}
opts := &kubernetes.ApiServerOptions{
Namespace: mizu.Config.MizuResourcesNamespace,
PodName: mizu.ApiServerPodName,
PodImage: mizu.Config.AgentImage,
ServiceAccountName: serviceAccountName,
IsNamespaceRestricted: mizu.Config.IsNsRestrictedMode(),
MizuApiFilteringOptions: mizuApiFilteringOptions,
MaxEntriesDBSizeBytes: mizu.Config.Tap.MaxEntriesDBSizeBytes(),
}
_, err = kubernetesProvider.CreateMizuApiServerPod(ctx, opts)
if err != nil {
fmt.Printf("Error creating mizu collector service: %v\n", err)
return err
}
mizu.Log.Debugf("Successfully created API server pod: %s", mizu.ApiServerPodName)
state.apiServerService, err = kubernetesProvider.CreateService(ctx, mizu.Config.MizuResourcesNamespace, mizu.ApiServerPodName, mizu.ApiServerPodName)
if err != nil {
return err
}
mizu.Log.Debugf("Successfully created service: %s", mizu.ApiServerPodName)
return nil
}
func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.TrafficFilteringOptions, error) {
func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) {
var compiledRegexSlice []*shared.SerializableRegexp
if tappingOptions.PlainTextFilterRegexes != nil && len(tappingOptions.PlainTextFilterRegexes) > 0 {
if mizu.Config.Tap.PlainTextFilterRegexes != nil && len(mizu.Config.Tap.PlainTextFilterRegexes) > 0 {
compiledRegexSlice = make([]*shared.SerializableRegexp, 0)
for _, regexStr := range tappingOptions.PlainTextFilterRegexes {
for _, regexStr := range mizu.Config.Tap.PlainTextFilterRegexes {
compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr)
if err != nil {
fmt.Printf("Regex %s is invalid: %v", regexStr, err)
return nil, err
}
compiledRegexSlice = append(compiledRegexSlice, compiledRegex)
}
}
return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: tappingOptions.HideHealthChecks}, nil
return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: mizu.Config.Tap.HideHealthChecks, DisableRedaction: mizu.Config.Tap.DisableRedaction}, nil
}
func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error {
func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string) error {
if len(nodeToTappedPodIPMap) > 0 {
var serviceAccountName string
if state.mizuServiceAccountExists {
serviceAccountName = mizu.ServiceAccountName
} else {
serviceAccountName = ""
}
if err := kubernetesProvider.ApplyMizuTapperDaemonSet(
ctx,
mizu.ResourcesNamespace,
mizu.Config.MizuResourcesNamespace,
mizu.TapperDaemonSetName,
tappingOptions.MizuImage,
mizu.Config.AgentImage,
mizu.TapperPodName,
fmt.Sprintf("%s.%s.svc.cluster.local", aggregatorService.Name, aggregatorService.Namespace),
fmt.Sprintf("%s.%s.svc.cluster.local", state.apiServerService.Name, state.apiServerService.Namespace),
nodeToTappedPodIPMap,
mizuServiceAccountExists,
tappingOptions.TapOutgoing,
serviceAccountName,
mizu.Config.Tap.TapOutgoing(),
); err != nil {
fmt.Printf("Error creating mizu tapper daemonset: %v\n", err)
return err
}
mizu.Log.Debugf("Successfully created %v tappers", len(nodeToTappedPodIPMap))
} else {
if err := kubernetesProvider.RemoveDaemonSet(ctx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
fmt.Printf("Error deleting mizu tapper daemonset: %v\n", err)
if err := kubernetesProvider.RemoveDaemonSet(ctx, mizu.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
return err
}
}
@ -156,41 +243,138 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
}
func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
fmt.Printf("\nRemoving mizu resources\n")
removalCtx, _ := context.WithTimeout(context.Background(), 5*time.Second)
if err := kubernetesProvider.RemovePod(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil {
fmt.Printf("Error removing Pod %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err)
removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
defer cancel()
if mizu.Config.DumpLogs {
mizuDir := mizu.GetMizuFolderPath()
filePath = path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05")))
if err := fsUtils.DumpLogs(kubernetesProvider, removalCtx, filePath); err != nil {
mizu.Log.Errorf("Failed dump logs %v", err)
}
}
if err := kubernetesProvider.RemoveService(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil {
fmt.Printf("Error removing Service %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err)
mizu.Log.Infof("\nRemoving mizu resources\n")
if !mizu.Config.IsNsRestrictedMode() {
if err := kubernetesProvider.RemoveNamespace(removalCtx, mizu.Config.MizuResourcesNamespace); err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Namespace %s: %v", mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
return
}
} else {
if err := kubernetesProvider.RemovePod(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Pod %s in namespace %s: %v", mizu.ApiServerPodName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
}
if err := kubernetesProvider.RemoveService(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service %s in namespace %s: %v", mizu.ApiServerPodName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
}
if err := kubernetesProvider.RemoveDaemonSet(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing DaemonSet %s in namespace %s: %v", mizu.TapperDaemonSetName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
}
if !state.doNotRemoveConfigMap {
if err := kubernetesProvider.RemoveConfigMap(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.ConfigMapName); err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing ConfigMap %s in namespace %s: %v", mizu.ConfigMapName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
}
}
}
if err := kubernetesProvider.RemoveDaemonSet(removalCtx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
fmt.Printf("Error removing DaemonSet %s in namespace %s: %s (%v,%+v)\n", mizu.TapperDaemonSetName, mizu.ResourcesNamespace, err, err, err)
if state.mizuServiceAccountExists {
if !mizu.Config.IsNsRestrictedMode() {
if err := kubernetesProvider.RemoveNonNamespacedResources(removalCtx, mizu.ClusterRoleName, mizu.ClusterRoleBindingName); err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing non-namespaced resources: %v", errormessage.FormatError(err)))
return
}
} else {
if err := kubernetesProvider.RemoveServicAccount(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.ServiceAccountName); err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service Account %s in namespace %s: %v", mizu.ServiceAccountName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
return
}
if err := kubernetesProvider.RemoveRole(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.RoleName); err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Role %s in namespace %s: %v", mizu.RoleName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
}
if err := kubernetesProvider.RemoveRoleBinding(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.RoleBindingName); err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing RoleBinding %s in namespace %s: %v", mizu.RoleBindingName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
}
}
}
if !mizu.Config.IsNsRestrictedMode() {
waitUntilNamespaceDeleted(removalCtx, cancel, kubernetesProvider)
}
}
func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, podRegex *regexp.Regexp, tappingOptions *MizuTapOptions) {
targetNamespace := getNamespace(tappingOptions, kubernetesProvider)
func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) {
// Call cancel if a terminating signal was received. Allows user to skip the wait.
go func() {
waitForFinish(ctx, cancel)
}()
added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, targetNamespace), podRegex)
if err := kubernetesProvider.WaitUtilNamespaceDeleted(ctx, mizu.Config.MizuResourcesNamespace); err != nil {
switch {
case ctx.Err() == context.Canceled:
// Do nothing. User interrupted the wait.
case err == wait.ErrWaitTimeout:
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Timeout while removing Namespace %s", mizu.Config.MizuResourcesNamespace))
default:
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error while waiting for Namespace %s to be deleted: %v", mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
}
}
}
func reportTappedPods() {
mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort)
tappedPodsUrl := fmt.Sprintf("http://%s/status/tappedPods", mizuProxiedUrl)
podInfos := make([]shared.PodInfo, 0)
for _, pod := range state.currentlyTappedPods {
podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace})
}
tapStatus := shared.TapStatus{Pods: podInfos}
if jsonValue, err := json.Marshal(tapStatus); err != nil {
mizu.Log.Debugf("[ERROR] failed Marshal the tapped pods %v", err)
} else {
if response, err := http.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
mizu.Log.Debugf("[ERROR] failed sending to API server the tapped pods %v", err)
} else if response.StatusCode != 200 {
mizu.Log.Debugf("[ERROR] failed sending to API server the tapped pods, response status code %v", response.StatusCode)
} else {
mizu.Log.Debugf("Reported to server API about %d taped pods successfully", len(podInfos))
}
}
}
func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, targetNamespaces []string, cancel context.CancelFunc) {
added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, targetNamespaces, mizu.Config.Tap.PodRegex())
restartTappers := func() {
if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegex, targetNamespace); err != nil {
fmt.Printf("Error getting pods by regex: %s (%v,%+v)\n", err, err, err)
cancel()
} else {
currentlyTappedPods = matchingPods
}
nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods)
err, changeFound := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespaces)
if err != nil {
fmt.Printf("Error building node to ips map: %s (%v,%+v)\n", err, err, err)
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Failed to update currently tapped pods: %v", err))
cancel()
}
if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
fmt.Printf("Error updating daemonset: %s (%v,%+v)\n", err, err, err)
if !changeFound {
mizu.Log.Debugf("Nothing changed update tappers not needed")
return
}
reportTappedPods()
nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
if err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error building node to ips map: %v", errormessage.FormatError(err)))
cancel()
}
if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating daemonset: %v", errormessage.FormatError(err)))
cancel()
}
}
@ -198,109 +382,187 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
for {
select {
case newTarget := <-added:
fmt.Printf(mizu.Green, fmt.Sprintf("+%s\n", newTarget.Name))
case removedTarget := <-removed:
fmt.Printf(mizu.Red, fmt.Sprintf("-%s\n", removedTarget.Name))
case pod := <-added:
mizu.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace)
restartTappersDebouncer.SetOn()
case modifiedTarget := <-modified:
case pod := <-removed:
mizu.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace)
restartTappersDebouncer.SetOn()
case pod := <-modified:
mizu.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP)
// Act only if the modified pod has already obtained an IP address.
// After filtering for IPs, on a normal pod restart this includes the following events:
// - Pod deletion
// - Pod reaches start state
// - Pod reaches ready state
// Ready/unready transitions might also trigger this event.
if modifiedTarget.Status.PodIP != "" {
if pod.Status.PodIP != "" {
restartTappersDebouncer.SetOn()
}
case <-errorChan:
case err := <-errorChan:
mizu.Log.Debugf("Watching pods loop, got error %v, stopping `restart tappers debouncer`", err)
restartTappersDebouncer.Cancel()
// TODO: Does this also perform cleanup?
cancel()
case <-ctx.Done():
mizu.Log.Debugf("Watching pods loop, context done, stopping `restart tappers debouncer`")
restartTappersDebouncer.Cancel()
return
}
}
}
func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.AggregatorPodName))
added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, mizu.ResourcesNamespace), podExactRegex)
func updateCurrentlyTappedPods(kubernetesProvider *kubernetes.Provider, ctx context.Context, targetNamespaces []string) (error, bool) {
changeFound := false
if matchingPods, err := kubernetesProvider.ListAllRunningPodsMatchingRegex(ctx, mizu.Config.Tap.PodRegex(), targetNamespaces); err != nil {
return err, false
} else {
podsToTap := excludeMizuPods(matchingPods)
addedPods, removedPods := getPodArrayDiff(state.currentlyTappedPods, podsToTap)
for _, addedPod := range addedPods {
changeFound = true
mizu.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", addedPod.Name))
}
for _, removedPod := range removedPods {
changeFound = true
mizu.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s", removedPod.Name))
}
state.currentlyTappedPods = podsToTap
}
return nil, changeFound
}
func excludeMizuPods(pods []core.Pod) []core.Pod {
mizuPrefixRegex := regexp.MustCompile("^" + mizu.MizuResourcesPrefix)
nonMizuPods := make([]core.Pod, 0)
for _, pod := range pods {
if !mizuPrefixRegex.MatchString(pod.Name) {
nonMizuPods = append(nonMizuPods, pod)
}
}
return nonMizuPods
}
func getPodArrayDiff(oldPods []core.Pod, newPods []core.Pod) (added []core.Pod, removed []core.Pod) {
added = getMissingPods(newPods, oldPods)
removed = getMissingPods(oldPods, newPods)
return added, removed
}
//returns pods present in pods1 array and missing in pods2 array
func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod {
missingPods := make([]core.Pod, 0)
for _, pod1 := range pods1 {
var found = false
for _, pod2 := range pods2 {
if pod1.UID == pod2.UID {
found = true
break
}
}
if !found {
missingPods = append(missingPods, pod1)
}
}
return missingPods
}
func createProxyToApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.ApiServerPodName))
added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{mizu.Config.MizuResourcesNamespace}, podExactRegex)
isPodReady := false
timeAfter := time.After(25 * time.Second)
for {
select {
case <-ctx.Done():
mizu.Log.Debugf("Watching API Server pod loop, ctx done")
return
case <-added:
mizu.Log.Debugf("Watching API Server pod loop, added")
continue
case <-removed:
fmt.Printf("%s removed\n", mizu.AggregatorPodName)
mizu.Log.Infof("%s removed", mizu.ApiServerPodName)
cancel()
return
case modifiedPod := <-modified:
if modifiedPod.Status.Phase == "Running" && !isPodReady {
isPodReady = true
go func() {
err := kubernetes.StartProxy(kubernetesProvider, tappingOptions.GuiPort, mizu.ResourcesNamespace, mizu.AggregatorPodName)
if err != nil {
fmt.Printf("Error occured while running k8s proxy %v\n", err)
cancel()
}
}()
mizuProxiedUrl := kubernetes.GetMizuCollectorProxiedHostAndPath(tappingOptions.GuiPort)
fmt.Printf("Mizu is available at http://%s\n", mizuProxiedUrl)
time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready
if tappingOptions.Analysis {
urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(tappingOptions.AnalysisDestination), tappingOptions.SleepIntervalSec)
u, err := url.ParseRequestURI(urlPath)
if err != nil {
log.Fatal(fmt.Sprintf("Failed parsing the URL %v\n", err))
}
rlog.Debugf("Sending get request to %v\n", u.String())
if response, err := http.Get(u.String()); err != nil || response.StatusCode != 200 {
fmt.Printf("error sending upload entries req, status code: %v, err: %v\n", response.StatusCode, err)
} else {
fmt.Printf(mizu.Purple, "Traffic is uploading to UP9 for further analsys")
fmt.Println()
}
}
if modifiedPod == nil {
mizu.Log.Debugf("Watching API Server pod loop, modifiedPod with nil")
continue
}
case <-time.After(25 * time.Second):
mizu.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase)
if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
isPodReady = true
go startProxyReportErrorIfAny(kubernetesProvider, cancel)
mizu.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort))
time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready
requestForAnalysis()
reportTappedPods()
}
case <-timeAfter:
if !isPodReady {
fmt.Printf("error: %s pod was not ready in time", mizu.AggregatorPodName)
mizu.Log.Errorf(uiUtils.Error, "Mizu API server was not ready in time")
cancel()
}
case <-errorChan:
mizu.Log.Debugf("[ERROR] Agent creation, watching %v namespace", mizu.Config.MizuResourcesNamespace)
cancel()
case <-ctx.Done():
return
}
}
}
func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
mizuRBACExists, err := kubernetesProvider.DoesMizuRBACExist(ctx, mizu.ResourcesNamespace)
func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
err := kubernetes.StartProxy(kubernetesProvider, mizu.Config.Tap.GuiPort, mizu.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
if err != nil {
fmt.Printf("warning: could not ensure mizu rbac resources exist %v\n", err)
return false
mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v\n"+
"Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName))
cancel()
}
if !mizuRBACExists {
err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.ResourcesNamespace, mizu.RBACVersion)
if err != nil {
fmt.Printf("warning: could not create mizu rbac resources %v\n", err)
return false
}
}
return true
}
func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) (map[string][]string, error) {
func requestForAnalysis() {
if !mizu.Config.Tap.Analysis {
return
}
mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort)
urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(mizu.Config.Tap.AnalysisDestination), mizu.Config.Tap.SleepIntervalSec)
u, parseErr := url.ParseRequestURI(urlPath)
if parseErr != nil {
mizu.Log.Fatal("Failed parsing the URL (consider changing the analysis dest URL), err: %v", parseErr)
}
mizu.Log.Debugf("Sending get request to %v", u.String())
if response, requestErr := http.Get(u.String()); requestErr != nil {
mizu.Log.Errorf("Failed to notify agent for analysis, err: %v", requestErr)
} else if response.StatusCode != 200 {
mizu.Log.Errorf("Failed to notify agent for analysis, status code: %v", response.StatusCode)
} else {
mizu.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis")
}
}
func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) (bool, error) {
if !mizu.Config.IsNsRestrictedMode() {
err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.Config.MizuResourcesNamespace, mizu.ServiceAccountName, mizu.ClusterRoleName, mizu.ClusterRoleBindingName, mizu.RBACVersion)
if err != nil {
return false, err
}
} else {
err := kubernetesProvider.CreateMizuRBACNamespaceRestricted(ctx, mizu.Config.MizuResourcesNamespace, mizu.ServiceAccountName, mizu.RoleName, mizu.RoleBindingName, mizu.RBACVersion)
if err != nil {
return false, err
}
}
return true, nil
}
func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) map[string][]string {
nodeToTappedPodIPMap := make(map[string][]string, 0)
for _, pod := range tappedPods {
existingList := nodeToTappedPodIPMap[pod.Spec.NodeName]
@ -310,7 +572,7 @@ func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) (map[string][]string, e
nodeToTappedPodIPMap[pod.Spec.NodeName] = append(nodeToTappedPodIPMap[pod.Spec.NodeName], pod.Status.PodIP)
}
}
return nodeToTappedPodIPMap, nil
return nodeToTappedPodIPMap
}
func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
@ -326,35 +588,12 @@ func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
}
}
func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
controlSocketStr := fmt.Sprintf("ws://%s/ws", kubernetes.GetMizuCollectorProxiedHostAndPath(tappingOptions.GuiPort))
controlSocket, err := mizu.CreateControlSocket(controlSocketStr)
if err != nil {
fmt.Printf("error establishing control socket connection %s\n", err)
cancel()
}
for {
select {
case <-ctx.Done():
return
default:
err = controlSocket.SendNewTappedPodsListMessage(currentlyTappedPods)
if err != nil {
rlog.Debugf("error Sending message via control socket %v, error: %s\n", controlSocketStr, err)
}
time.Sleep(10 * time.Second)
}
}
}
func getNamespace(tappingOptions *MizuTapOptions, kubernetesProvider *kubernetes.Provider) string {
if tappingOptions.AllNamespaces {
return mizu.K8sAllNamespaces
} else if len(tappingOptions.Namespace) > 0 {
return tappingOptions.Namespace
func getNamespaces(kubernetesProvider *kubernetes.Provider) []string {
if mizu.Config.Tap.AllNamespaces {
return []string{mizu.K8sAllNamespaces}
} else if len(mizu.Config.Tap.Namespaces) > 0 {
return mizu.Config.Tap.Namespaces
} else {
return kubernetesProvider.CurrentNamespace()
return []string{kubernetesProvider.CurrentNamespace()}
}
}

26
cli/cmd/version.go
View File

@ -1,32 +1,27 @@
package cmd
import (
"fmt"
"github.com/up9inc/mizu/cli/mizu"
"strconv"
"time"
"github.com/creasty/defaults"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/mizu/configStructs"
)
type MizuVersionOptions struct {
DebugInfo bool
}
var mizuVersionOptions = &MizuVersionOptions{}
var versionCmd = &cobra.Command{
Use: "version",
Short: "Print version info",
RunE: func(cmd *cobra.Command, args []string) error {
if mizuVersionOptions.DebugInfo {
go mizu.ReportRun("version", mizu.Config.Version)
if mizu.Config.Version.DebugInfo {
timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
fmt.Printf("Version: %s \nBranch: %s (%s) \n", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
fmt.Printf("Build Time: %s (%s)\n", mizu.BuildTimestamp, time.Unix(timeStampInt, 0))
mizu.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
mizu.Log.Infof("Build Time: %s (%s)", mizu.BuildTimestamp, time.Unix(timeStampInt, 0))
} else {
fmt.Printf("Version: %s (%s)\n", mizu.SemVer, mizu.Branch)
mizu.Log.Infof("Version: %s (%s)", mizu.SemVer, mizu.Branch)
}
return nil
},
@ -35,6 +30,9 @@ var versionCmd = &cobra.Command{
func init() {
rootCmd.AddCommand(versionCmd)
versionCmd.Flags().BoolVarP(&mizuVersionOptions.DebugInfo, "debug", "d", false, "Provide all information about version")
defaultVersionConfig := configStructs.VersionConfig{}
defaults.Set(&defaultVersionConfig)
versionCmd.Flags().BoolP(configStructs.DebugInfoVersionName, "d", defaultVersionConfig.DebugInfo, "Provide all information about version")
}

22
cli/cmd/view.go
View File

@ -1,34 +1,28 @@
package cmd
import (
"github.com/creasty/defaults"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/mizu/configStructs"
)
type MizuViewOptions struct {
GuiPort uint16
}
var mizuViewOptions = &MizuViewOptions{}
var viewCmd = &cobra.Command{
Use: "view",
Short: "Open GUI in browser",
RunE: func(cmd *cobra.Command, args []string) error {
if isCompatible, err := mizu.CheckVersionCompatibility(mizuFetchOptions.MizuPort); err != nil {
return err
} else if !isCompatible {
return nil
}
runMizuView(mizuViewOptions)
go mizu.ReportRun("view", mizu.Config.View)
runMizuView()
return nil
},
}
func init() {
rootCmd.AddCommand(viewCmd)
viewCmd.Flags().Uint16VarP(&mizuViewOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
defaultViewConfig := configStructs.ViewConfig{}
defaults.Set(&defaultViewConfig)
viewCmd.Flags().Uint16P(configStructs.GuiPortViewName, "p", defaultViewConfig.GuiPort, "Provide a custom port for the web interface webserver")
viewCmd.Flags().StringP(configStructs.KubeConfigPathViewName, "k", defaultViewConfig.KubeConfigPath, "Path to kube-config file")
}

39
cli/cmd/viewRunner.go
View File

@ -8,32 +8,47 @@ import (
"net/http"
)
func runMizuView(mizuViewOptions *MizuViewOptions) {
kubernetesProvider := kubernetes.NewProvider("")
func runMizuView() {
kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.View.KubeConfigPath)
if err != nil {
mizu.Log.Error(err)
return
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
exists, err := kubernetesProvider.DoesServicesExist(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName)
exists, err := kubernetesProvider.DoesServicesExist(ctx, mizu.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
if err != nil {
panic(err)
mizu.Log.Errorf("Failed to found mizu service %v", err)
cancel()
return
}
if !exists {
fmt.Printf("The %s service not found\n", mizu.AggregatorPodName)
mizu.Log.Infof("%s service not found, you should run `mizu tap` command first", mizu.ApiServerPodName)
cancel()
return
}
mizuProxiedUrl := kubernetes.GetMizuCollectorProxiedHostAndPath(mizuViewOptions.GuiPort)
mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.View.GuiPort)
_, err = http.Get(fmt.Sprintf("http://%s/", mizuProxiedUrl))
if err == nil {
fmt.Printf("Found a running service %s and open port %d\n", mizu.AggregatorPodName, mizuViewOptions.GuiPort)
mizu.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, mizu.Config.View.GuiPort)
return
}
fmt.Printf("Found service %s, creating k8s proxy\n", mizu.AggregatorPodName)
mizu.Log.Debugf("Found service %s, creating k8s proxy", mizu.ApiServerPodName)
fmt.Printf("Mizu is available at http://%s\n", kubernetes.GetMizuCollectorProxiedHostAndPath(mizuViewOptions.GuiPort))
err = kubernetes.StartProxy(kubernetesProvider, mizuViewOptions.GuiPort, mizu.ResourcesNamespace, mizu.AggregatorPodName)
if err != nil {
fmt.Printf("Error occured while running k8s proxy %v\n", err)
go startProxyReportErrorIfAny(kubernetesProvider, cancel)
mizu.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.View.GuiPort))
if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.View.GuiPort); err != nil {
mizu.Log.Errorf("Failed to check versions compatibility %v", err)
cancel()
return
} else if !isCompatible {
cancel()
return
}
waitForFinish(ctx, cancel)
}

38
cli/errormessage/errormessage.go Normal file
View File

@ -0,0 +1,38 @@
package errormessage
import (
"errors"
"fmt"
"github.com/up9inc/mizu/cli/mizu"
regexpsyntax "regexp/syntax"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
)
// formatError wraps error with a detailed message that is meant for the user.
// While the errors are meant to be displayed, they are not meant to be exported as classes outsite of CLI.
func FormatError(err error) error {
var errorNew error
if k8serrors.IsForbidden(err) {
errorNew = fmt.Errorf("insufficient permissions: %w. "+
"supply the required permission or control Mizu's access to namespaces by setting %s "+
"in the config file or setting the tapped namespace with --%s %s=<NAMEPSACE>",
err,
mizu.MizuResourcesNamespaceConfigName,
mizu.SetCommandName,
mizu.MizuResourcesNamespaceConfigName)
} else if syntaxError, isSyntaxError := asRegexSyntaxError(err); isSyntaxError {
errorNew = fmt.Errorf("regex %s is invalid: %w", syntaxError.Expr, err)
} else {
errorNew = err
}
return errorNew
}
func asRegexSyntaxError(err error) (*regexpsyntax.Error, bool) {
var syntaxError *regexpsyntax.Error
return syntaxError, errors.As(err, &syntaxError)
}

26
cli/fsUtils/dirUtils.go Normal file
View File

@ -0,0 +1,26 @@
package fsUtils
import (
"errors"
"fmt"
"os"
)
func EnsureDir(dirName string) error {
err := os.Mkdir(dirName, 0700)
if err == nil {
return nil
}
if os.IsExist(err) {
// check that the existing path is a directory
info, err := os.Stat(dirName)
if err != nil {
return err
}
if !info.IsDir() {
return errors.New(fmt.Sprintf("path exists but is not a directory: %s", dirName))
}
return nil
}
return err
}

58
cli/fsUtils/mizuLogsUtils.go Normal file
View File

@ -0,0 +1,58 @@
package fsUtils
import (
"archive/zip"
"context"
"fmt"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"os"
"regexp"
)
func DumpLogs(provider *kubernetes.Provider, ctx context.Context, filePath string) error {
podExactRegex := regexp.MustCompile("^" + mizu.MizuResourcesPrefix)
pods, err := provider.ListAllPodsMatchingRegex(ctx, podExactRegex, []string{mizu.Config.MizuResourcesNamespace})
if err != nil {
return err
}
if len(pods) == 0 {
return fmt.Errorf("no mizu pods found in namespace %s", mizu.Config.MizuResourcesNamespace)
}
newZipFile, err := os.Create(filePath)
if err != nil {
return err
}
defer newZipFile.Close()
zipWriter := zip.NewWriter(newZipFile)
defer zipWriter.Close()
for _, pod := range pods {
logs, err := provider.GetPodLogs(pod.Namespace, pod.Name, ctx)
if err != nil {
mizu.Log.Errorf("Failed to get logs, %v", err)
continue
} else {
mizu.Log.Debugf("Successfully read log length %d for pod: %s.%s", len(logs), pod.Namespace, pod.Name)
}
if err := AddStrToZip(zipWriter, logs, fmt.Sprintf("%s.%s.log", pod.Namespace, pod.Name)); err != nil {
mizu.Log.Errorf("Failed write logs, %v", err)
} else {
mizu.Log.Infof("Successfully added log length %d from pod: %s.%s", len(logs), pod.Namespace, pod.Name)
}
}
if err := AddFileToZip(zipWriter, mizu.GetConfigFilePath()); err != nil {
mizu.Log.Debugf("Failed write file, %v", err)
} else {
mizu.Log.Infof("Successfully added file %s", mizu.GetConfigFilePath())
}
if err := AddFileToZip(zipWriter, mizu.GetLogFilePath()); err != nil {
mizu.Log.Debugf("Failed write file, %v", err)
} else {
mizu.Log.Infof("Successfully added file %s", mizu.GetLogFilePath())
}
mizu.Log.Infof("You can find the zip with all logs in %s\n", filePath)
return nil
}

55
cli/fsUtils/zipUtils.go Normal file
View File

@ -0,0 +1,55 @@
package fsUtils
import (
"archive/zip"
"fmt"
"io"
"os"
"path/filepath"
)
func AddFileToZip(zipWriter *zip.Writer, filename string) error {
fileToZip, err := os.Open(filename)
if err != nil {
return fmt.Errorf("failed to open file %s, %w", filename, err)
}
defer fileToZip.Close()
// Get the file information
info, err := fileToZip.Stat()
if err != nil {
return fmt.Errorf("failed to get file information %s, %w", filename, err)
}
header, err := zip.FileInfoHeader(info)
if err != nil {
return err
}
// Using FileInfoHeader() above only uses the basename of the file. If we want
// to preserve the folder structure we can overwrite this with the full path.
header.Name = filepath.Base(filename)
// Change to deflate to gain better compression
// see http://golang.org/pkg/archive/zip/#pkg-constants
header.Method = zip.Deflate
writer, err := zipWriter.CreateHeader(header)
if err != nil {
return fmt.Errorf("failed to create header in zip for %s, %w", filename, err)
}
_, err = io.Copy(writer, fileToZip)
return err
}
func AddStrToZip(writer *zip.Writer, logs string, fileName string) error {
if zipFile, err := writer.Create(fileName); err != nil {
return fmt.Errorf("couldn't create a log file inside zip for %s, %w", fileName, err)
} else {
if _, err = zipFile.Write([]byte(logs)); err != nil {
return fmt.Errorf("couldn't write logs to zip file: %s, %w", fileName, err)
}
}
return nil
}

6
cli/go.mod
View File

@ -3,10 +3,14 @@ module github.com/up9inc/mizu/cli
go 1.16
require (
github.com/creasty/defaults v1.5.1
github.com/google/go-github/v37 v37.0.0
github.com/gorilla/websocket v1.4.2
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
github.com/spf13/cobra v1.1.3
github.com/spf13/pflag v1.0.5
github.com/up9inc/mizu/shared v0.0.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
k8s.io/api v0.21.2
k8s.io/apimachinery v0.21.2
k8s.io/client-go v0.21.2

25
cli/go.sum
View File

@ -82,6 +82,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creasty/defaults v1.5.1 h1:j8WexcS3d/t4ZmllX4GEkl4wIB/trOr035ajcLHCISM=
github.com/creasty/defaults v1.5.1/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -99,6 +101,7 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
@ -174,6 +177,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@ -203,11 +207,17 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github/v37 v37.0.0 h1:rCspN8/6kB1BAJWZfuafvHhyfIo5fkAulaP/3bOQ/tM=
github.com/google/go-github/v37 v37.0.0/go.mod h1:LM7in3NmXDrX58GbEHy7FtNLbI2JijX93RnMKvWG3m4=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@ -218,6 +228,7 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
@ -245,6 +256,7 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
@ -324,6 +336,8 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
@ -331,6 +345,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@ -355,8 +370,6 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@ -398,6 +411,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@ -679,8 +694,9 @@ gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@ -705,6 +721,7 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
k8s.io/kubectl v0.21.2 h1:9XPCetvOMDqrIZZXb1Ei+g8t6KrIp9ENJaysQjUuLiE=
k8s.io/kubectl v0.21.2/go.mod h1:PgeUclpG8VVmmQIl8zpLar3IQEpFc9mrmvlwY3CK1xo=

25
cli/goUtils/funcWrappers.go Normal file
View File

@ -0,0 +1,25 @@
package goUtils
import (
"github.com/up9inc/mizu/cli/mizu"
"reflect"
"runtime/debug"
)
func HandleExcWrapper(fn interface{}, params ...interface{}) (result []reflect.Value) {
defer func() {
if panicMessage := recover(); panicMessage != nil {
stack := debug.Stack()
mizu.Log.Fatalf("Unhandled panic: %v\n stack: %s", panicMessage, stack)
}
}()
f := reflect.ValueOf(fn)
if f.Type().NumIn() != len(params) {
panic("incorrect number of parameters!")
}
inputs := make([]reflect.Value, len(params))
for k, in := range params {
inputs[k] = reflect.ValueOf(in)
}
return f.Call(inputs)
}

511
cli/kubernetes/provider.go
View File

@ -1,21 +1,26 @@
package kubernetes
import (
"bytes"
_ "bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strconv"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/shared"
"io"
core "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
resource "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/watch"
applyconfapp "k8s.io/client-go/applyconfigurations/apps/v1"
@ -27,8 +32,10 @@ import (
_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/clientcmd"
_ "k8s.io/client-go/tools/portforward"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/client-go/util/homedir"
)
@ -40,15 +47,19 @@ type Provider struct {
}
const (
serviceAccountName = "mizu-service-account"
fieldManagerName = "mizu-manager"
fieldManagerName = "mizu-manager"
)
func NewProvider(kubeConfigPath string) *Provider {
func NewProvider(kubeConfigPath string) (*Provider, error) {
kubernetesConfig := loadKubernetesConfiguration(kubeConfigPath)
restClientConfig, err := kubernetesConfig.ClientConfig()
if err != nil {
panic(err.Error())
if clientcmd.IsEmptyConfig(err) {
return nil, fmt.Errorf("Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'\n")
}
if clientcmd.IsConfigurationInvalid(err) {
return nil, fmt.Errorf("Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'\n")
}
}
clientSet := getClientSet(restClientConfig)
@ -56,7 +67,7 @@ func NewProvider(kubeConfigPath string) *Provider {
clientSet: clientSet,
kubernetesConfig: kubernetesConfig,
clientConfig: *restClientConfig,
}
}, nil
}
func (provider *Provider) CurrentNamespace() string {
@ -64,6 +75,46 @@ func (provider *Provider) CurrentNamespace() string {
return ns
}
func (provider *Provider) WaitUtilNamespaceDeleted(ctx context.Context, name string) error {
fieldSelector := fmt.Sprintf("metadata.name=%s", name)
var limit int64 = 1
lw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
options.FieldSelector = fieldSelector
options.Limit = limit
return provider.clientSet.CoreV1().Namespaces().List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
options.FieldSelector = fieldSelector
options.Limit = limit
return provider.clientSet.CoreV1().Namespaces().Watch(ctx, options)
},
}
var preconditionFunc watchtools.PreconditionFunc = func(store cache.Store) (bool, error) {
_, exists, err := store.Get(&core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}})
if err != nil {
return false, err
}
if exists {
return false, nil
}
return true, nil
}
conditionFunc := func(e watch.Event) (bool, error) {
if e.Type == watch.Deleted {
return true, nil
}
return false, nil
}
obj := &core.Namespace{}
_, err := watchtools.UntilWithSync(ctx, lw, obj, preconditionFunc, conditionFunc)
return err
}
func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) watch.Interface {
watcher, err := provider.clientSet.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{Watch: true})
if err != nil {
@ -72,42 +123,76 @@ func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) w
return watcher
}
func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace string, podName string, podImage string, linkServiceAccount bool, mizuApiFilteringOptions *shared.TrafficFilteringOptions, maxEntriesDBSizeBytes int64) (*core.Pod, error) {
marshaledFilteringOptions, err := json.Marshal(mizuApiFilteringOptions)
func (provider *Provider) CreateNamespace(ctx context.Context, name string) (*core.Namespace, error) {
namespaceSpec := &core.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
}
return provider.clientSet.CoreV1().Namespaces().Create(ctx, namespaceSpec, metav1.CreateOptions{})
}
type ApiServerOptions struct {
Namespace string
PodName string
PodImage string
ServiceAccountName string
IsNamespaceRestricted bool
MizuApiFilteringOptions *shared.TrafficFilteringOptions
MaxEntriesDBSizeBytes int64
}
func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiServerOptions) (*core.Pod, error) {
marshaledFilteringOptions, err := json.Marshal(opts.MizuApiFilteringOptions)
if err != nil {
return nil, err
}
configMapVolumeName := &core.ConfigMapVolumeSource{}
configMapVolumeName.Name = mizu.ConfigMapName
configMapOptional := true
configMapVolumeName.Optional = &configMapOptional
cpuLimit, err := resource.ParseQuantity("750m")
if err != nil {
return nil, errors.New("invalid cpu limit for aggregator container")
return nil, errors.New(fmt.Sprintf("invalid cpu limit for %s container", opts.PodName))
}
memLimit, err := resource.ParseQuantity("512Mi")
if err != nil {
return nil, errors.New("invalid memory limit for aggregator container")
return nil, errors.New(fmt.Sprintf("invalid memory limit for %s container", opts.PodName))
}
cpuRequests, err := resource.ParseQuantity("50m")
if err != nil {
return nil, errors.New("invalid cpu request for aggregator container")
return nil, errors.New(fmt.Sprintf("invalid cpu request for %s container", opts.PodName))
}
memRequests, err := resource.ParseQuantity("50Mi")
if err != nil {
return nil, errors.New("invalid memory request for aggregator container")
return nil, errors.New(fmt.Sprintf("invalid memory request for %s container", opts.PodName))
}
command := []string{"./mizuagent", "--api-server"}
if opts.IsNamespaceRestricted {
command = append(command, "--namespace", opts.Namespace)
}
pod := &core.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Namespace: namespace,
Labels: map[string]string{"app": podName},
Name: opts.PodName,
Namespace: opts.Namespace,
Labels: map[string]string{"app": opts.PodName},
},
Spec: core.PodSpec{
Containers: []core.Container{
{
Name: podName,
Image: podImage,
Name: opts.PodName,
Image: opts.PodImage,
ImagePullPolicy: core.PullAlways,
Command: []string{"./mizuagent", "--aggregator"},
VolumeMounts: []core.VolumeMount{
{
Name: mizu.ConfigMapName,
MountPath: shared.RulePolicyPath,
},
},
Command: command,
Env: []core.EnvVar{
{
Name: shared.HostModeEnvVar,
@ -118,31 +203,39 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace
Value: string(marshaledFilteringOptions),
},
{
Name: shared.MaxEntriesDBSizeByteSEnvVar,
Value: strconv.FormatInt(maxEntriesDBSizeBytes, 10),
Name: shared.MaxEntriesDBSizeBytesEnvVar,
Value: strconv.FormatInt(opts.MaxEntriesDBSizeBytes, 10),
},
},
Resources: core.ResourceRequirements{
Limits: core.ResourceList{
"cpu": cpuLimit,
"cpu": cpuLimit,
"memory": memLimit,
},
Requests: core.ResourceList{
"cpu": cpuRequests,
"cpu": cpuRequests,
"memory": memRequests,
},
},
},
},
Volumes: []core.Volume{
{
Name: mizu.ConfigMapName,
VolumeSource: core.VolumeSource{
ConfigMap: configMapVolumeName,
},
},
},
DNSPolicy: core.DNSClusterFirstWithHostNet,
TerminationGracePeriodSeconds: new(int64),
},
}
//define the service account only when it exists to prevent pod crash
if linkServiceAccount {
pod.Spec.ServiceAccountName = serviceAccountName
if opts.ServiceAccountName != "" {
pod.Spec.ServiceAccountName = opts.ServiceAccountName
}
return provider.clientSet.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
return provider.clientSet.CoreV1().Pods(opts.Namespace).Create(ctx, pod, metav1.CreateOptions{})
}
func (provider *Provider) CreateService(ctx context.Context, namespace string, serviceName string, appLabelValue string) (*core.Service, error) {
@ -160,9 +253,57 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s
return provider.clientSet.CoreV1().Services(namespace).Create(ctx, &service, metav1.CreateOptions{})
}
func (provider *Provider) DoesMizuRBACExist(ctx context.Context, namespace string) (bool, error) {
func (provider *Provider) DoesServiceAccountExist(ctx context.Context, namespace string, serviceAccountName string) (bool, error) {
serviceAccount, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccountName, metav1.GetOptions{})
return provider.doesResourceExist(serviceAccount, err)
}
func (provider *Provider) DoesConfigMapExist(ctx context.Context, namespace string, name string) (bool, error) {
resource, err := provider.clientSet.CoreV1().ConfigMaps(namespace).Get(ctx, name, metav1.GetOptions{})
return provider.doesResourceExist(resource, err)
}
func (provider *Provider) DoesServicesExist(ctx context.Context, namespace string, name string) (bool, error) {
resource, err := provider.clientSet.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{})
return provider.doesResourceExist(resource, err)
}
func (provider *Provider) DoesNamespaceExist(ctx context.Context, name string) (bool, error) {
resource, err := provider.clientSet.CoreV1().Namespaces().Get(ctx, name, metav1.GetOptions{})
return provider.doesResourceExist(resource, err)
}
func (provider *Provider) DoesClusterRoleExist(ctx context.Context, name string) (bool, error) {
resource, err := provider.clientSet.RbacV1().ClusterRoles().Get(ctx, name, metav1.GetOptions{})
return provider.doesResourceExist(resource, err)
}
func (provider *Provider) DoesClusterRoleBindingExist(ctx context.Context, name string) (bool, error) {
resource, err := provider.clientSet.RbacV1().ClusterRoleBindings().Get(ctx, name, metav1.GetOptions{})
return provider.doesResourceExist(resource, err)
}
func (provider *Provider) DoesRoleExist(ctx context.Context, namespace string, name string) (bool, error) {
resource, err := provider.clientSet.RbacV1().Roles(namespace).Get(ctx, name, metav1.GetOptions{})
return provider.doesResourceExist(resource, err)
}
func (provider *Provider) DoesRoleBindingExist(ctx context.Context, namespace string, name string) (bool, error) {
resource, err := provider.clientSet.RbacV1().RoleBindings(namespace).Get(ctx, name, metav1.GetOptions{})
return provider.doesResourceExist(resource, err)
}
func (provider *Provider) DoesPodExist(ctx context.Context, namespace string, name string) (bool, error) {
resource, err := provider.clientSet.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
return provider.doesResourceExist(resource, err)
}
func (provider *Provider) DoesDaemonSetExist(ctx context.Context, namespace string, name string) (bool, error) {
resource, err := provider.clientSet.AppsV1().DaemonSets(namespace).Get(ctx, name, metav1.GetOptions{})
return provider.doesResourceExist(resource, err)
}
func (provider *Provider) doesResourceExist(resource interface{}, err error) (bool, error) {
var statusError *k8serrors.StatusError
if errors.As(err, &statusError) {
// expected behavior when resource does not exist
@ -173,27 +314,10 @@ func (provider *Provider) DoesMizuRBACExist(ctx context.Context, namespace strin
if err != nil {
return false, err
}
return serviceAccount != nil, nil
return resource != nil, nil
}
func (provider *Provider) DoesServicesExist(ctx context.Context, namespace string, serviceName string) (bool, error) {
service, err := provider.clientSet.CoreV1().Services(namespace).Get(ctx, serviceName, metav1.GetOptions{})
var statusError *k8serrors.StatusError
if errors.As(err, &statusError) {
if statusError.ErrStatus.Reason == metav1.StatusReasonNotFound {
return false, nil
}
}
if err != nil {
return false, err
}
return service != nil, nil
}
func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, version string) error {
clusterRoleName := "mizu-cluster-role"
func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, serviceAccountName string, clusterRoleName string, clusterRoleBindingName string, version string) error {
serviceAccount := &core.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: serviceAccountName,
@ -216,7 +340,7 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string,
}
clusterRoleBinding := &rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "mizu-cluster-role-binding",
Name: clusterRoleBindingName,
Labels: map[string]string{"mizu-cli-version": version},
},
RoleRef: rbac.RoleRef{
@ -233,23 +357,148 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string,
},
}
_, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Create(ctx, serviceAccount, metav1.CreateOptions{})
if err != nil {
if err != nil && !k8serrors.IsAlreadyExists(err) {
return err
}
_, err = provider.clientSet.RbacV1().ClusterRoles().Create(ctx, clusterRole, metav1.CreateOptions{})
if err != nil {
if err != nil && !k8serrors.IsAlreadyExists(err) {
return err
}
_, err = provider.clientSet.RbacV1().ClusterRoleBindings().Create(ctx, clusterRoleBinding, metav1.CreateOptions{})
if err != nil {
if err != nil && !k8serrors.IsAlreadyExists(err) {
return err
}
return nil
}
func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context, namespace string, serviceAccountName string, roleName string, roleBindingName string, version string) error {
serviceAccount := &core.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: serviceAccountName,
Namespace: namespace,
Labels: map[string]string{"mizu-cli-version": version},
},
}
role := &rbac.Role{
ObjectMeta: metav1.ObjectMeta{
Name: roleName,
Labels: map[string]string{"mizu-cli-version": version},
},
Rules: []rbac.PolicyRule{
{
APIGroups: []string{"", "extensions", "apps"},
Resources: []string{"pods", "services", "endpoints"},
Verbs: []string{"list", "get", "watch"},
},
},
}
roleBinding := &rbac.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: roleBindingName,
Labels: map[string]string{"mizu-cli-version": version},
},
RoleRef: rbac.RoleRef{
Name: roleName,
Kind: "Role",
APIGroup: "rbac.authorization.k8s.io",
},
Subjects: []rbac.Subject{
{
Kind: "ServiceAccount",
Name: serviceAccountName,
Namespace: namespace,
},
},
}
_, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Create(ctx, serviceAccount, metav1.CreateOptions{})
if err != nil && !k8serrors.IsAlreadyExists(err) {
return err
}
_, err = provider.clientSet.RbacV1().Roles(namespace).Create(ctx, role, metav1.CreateOptions{})
if err != nil && !k8serrors.IsAlreadyExists(err) {
return err
}
_, err = provider.clientSet.RbacV1().RoleBindings(namespace).Create(ctx, roleBinding, metav1.CreateOptions{})
if err != nil && !k8serrors.IsAlreadyExists(err) {
return err
}
return nil
}
func (provider *Provider) RemoveNamespace(ctx context.Context, name string) error {
if isFound, err := provider.DoesNamespaceExist(ctx, name); err != nil {
return err
} else if !isFound {
return nil
}
return provider.clientSet.CoreV1().Namespaces().Delete(ctx, name, metav1.DeleteOptions{})
}
func (provider *Provider) RemoveNonNamespacedResources(ctx context.Context, clusterRoleName string, clusterRoleBindingName string) error {
if err := provider.RemoveClusterRole(ctx, clusterRoleName); err != nil {
return err
}
if err := provider.RemoveClusterRoleBinding(ctx, clusterRoleBindingName); err != nil {
return err
}
return nil
}
func (provider *Provider) RemoveClusterRole(ctx context.Context, name string) error {
if isFound, err := provider.DoesClusterRoleExist(ctx, name); err != nil {
return err
} else if !isFound {
return nil
}
return provider.clientSet.RbacV1().ClusterRoles().Delete(ctx, name, metav1.DeleteOptions{})
}
func (provider *Provider) RemoveClusterRoleBinding(ctx context.Context, name string) error {
if isFound, err := provider.DoesClusterRoleBindingExist(ctx, name); err != nil {
return err
} else if !isFound {
return nil
}
return provider.clientSet.RbacV1().ClusterRoleBindings().Delete(ctx, name, metav1.DeleteOptions{})
}
func (provider *Provider) RemoveRoleBinding(ctx context.Context, namespace string, name string) error {
if isFound, err := provider.DoesRoleBindingExist(ctx, namespace, name); err != nil {
return err
} else if !isFound {
return nil
}
return provider.clientSet.RbacV1().RoleBindings(namespace).Delete(ctx, name, metav1.DeleteOptions{})
}
func (provider *Provider) RemoveRole(ctx context.Context, namespace string, name string) error {
if isFound, err := provider.DoesRoleExist(ctx, namespace, name); err != nil {
return err
} else if !isFound {
return nil
}
return provider.clientSet.RbacV1().Roles(namespace).Delete(ctx, name, metav1.DeleteOptions{})
}
func (provider *Provider) RemoveServicAccount(ctx context.Context, namespace string, name string) error {
if isFound, err := provider.DoesServiceAccountExist(ctx, namespace, name); err != nil {
return err
} else if !isFound {
return nil
}
return provider.clientSet.CoreV1().ServiceAccounts(namespace).Delete(ctx, name, metav1.DeleteOptions{})
}
func (provider *Provider) RemovePod(ctx context.Context, namespace string, podName string) error {
if isFound, err := provider.CheckPodExists(ctx, namespace, podName);
err != nil {
if isFound, err := provider.DoesPodExist(ctx, namespace, podName); err != nil {
return err
} else if !isFound {
return nil
@ -258,9 +507,18 @@ func (provider *Provider) RemovePod(ctx context.Context, namespace string, podNa
return provider.clientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{})
}
func (provider *Provider) RemoveConfigMap(ctx context.Context, namespace string, configMapName string) error {
if isFound, err := provider.DoesConfigMapExist(ctx, namespace, configMapName); err != nil {
return err
} else if !isFound {
return nil
}
return provider.clientSet.CoreV1().ConfigMaps(namespace).Delete(ctx, configMapName, metav1.DeleteOptions{})
}
func (provider *Provider) RemoveService(ctx context.Context, namespace string, serviceName string) error {
if isFound, err := provider.CheckServiceExists(ctx, namespace, serviceName);
err != nil {
if isFound, err := provider.DoesServicesExist(ctx, namespace, serviceName); err != nil {
return err
} else if !isFound {
return nil
@ -270,8 +528,7 @@ func (provider *Provider) RemoveService(ctx context.Context, namespace string, s
}
func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, daemonSetName string) error {
if isFound, err := provider.CheckDaemonSetExists(ctx, namespace, daemonSetName);
err != nil {
if isFound, err := provider.DoesDaemonSetExist(ctx, namespace, daemonSetName); err != nil {
return err
} else if !isFound {
return nil
@ -280,58 +537,33 @@ func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string,
return provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{})
}
func (provider *Provider) CheckPodExists(ctx context.Context, namespace string, name string) (bool, error) {
listOptions := metav1.ListOptions{
FieldSelector: fmt.Sprintf("metadata.name=%s", name),
Limit: 1,
}
resourceList, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, listOptions)
if err != nil {
return false, err
func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, data string) error {
if data == "" {
return nil
}
if len(resourceList.Items) > 0 {
return true, nil
configMapData := make(map[string]string, 0)
configMapData[shared.RulePolicyFileName] = data
configMap := &core.ConfigMap{
TypeMeta: metav1.TypeMeta{
Kind: "ConfigMap",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: configMapName,
Namespace: namespace,
},
Data: configMapData,
}
return false, nil
if _, err := provider.clientSet.CoreV1().ConfigMaps(namespace).Create(ctx, configMap, metav1.CreateOptions{}); err != nil {
return err
}
return nil
}
func (provider *Provider) CheckServiceExists(ctx context.Context, namespace string, name string) (bool, error) {
listOptions := metav1.ListOptions{
FieldSelector: fmt.Sprintf("metadata.name=%s", name),
Limit: 1,
}
resourceList, err := provider.clientSet.CoreV1().Services(namespace).List(ctx, listOptions)
if err != nil {
return false, err
}
func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool) error {
mizu.Log.Debugf("Applying %d tapper deamonsets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
if len(resourceList.Items) > 0 {
return true, nil
}
return false, nil
}
func (provider *Provider) CheckDaemonSetExists(ctx context.Context, namespace string, name string) (bool, error) {
listOptions := metav1.ListOptions{
FieldSelector: fmt.Sprintf("metadata.name=%s", name),
Limit: 1,
}
resourceList, err := provider.clientSet.AppsV1().DaemonSets(namespace).List(ctx, listOptions)
if err != nil {
return false, err
}
if len(resourceList.Items) > 0 {
return true, nil
}
return false, nil
}
func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, aggregatorPodIp string, nodeToTappedPodIPMap map[string][]string, linkServiceAccount bool, tapOutgoing bool) error {
if len(nodeToTappedPodIPMap) == 0 {
return fmt.Errorf("Daemon set %s must tap at least 1 pod", daemonSetName)
}
@ -346,18 +578,17 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
"-i", "any",
"--tap",
"--hardump",
"--aggregator-address", fmt.Sprintf("ws://%s/wsTapper", aggregatorPodIp),
"--api-server-address", fmt.Sprintf("ws://%s/wsTapper", apiServerPodIp),
}
if tapOutgoing {
mizuCmd = append(mizuCmd, "--anydirection")
}
privileged := true
agentContainer := applyconfcore.Container()
agentContainer.WithName(tapperPodName)
agentContainer.WithImage(podImage)
agentContainer.WithImagePullPolicy(core.PullAlways)
agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(privileged))
agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(true))
agentContainer.WithCommand(mizuCmd...)
agentContainer.WithEnv(
applyconfcore.EnvVar().WithName(shared.HostModeEnvVar).WithValue("1"),
@ -372,26 +603,26 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
)
cpuLimit, err := resource.ParseQuantity("500m")
if err != nil {
return errors.New("invalid cpu limit for tapper container")
return errors.New(fmt.Sprintf("invalid cpu limit for %s container", tapperPodName))
}
memLimit, err := resource.ParseQuantity("1Gi")
if err != nil {
return errors.New("invalid memory limit for tapper container")
return errors.New(fmt.Sprintf("invalid memory limit for %s container", tapperPodName))
}
cpuRequests, err := resource.ParseQuantity("50m")
if err != nil {
return errors.New("invalid cpu request for tapper container")
return errors.New(fmt.Sprintf("invalid cpu request for %s container", tapperPodName))
}
memRequests, err := resource.ParseQuantity("50Mi")
if err != nil {
return errors.New("invalid memory request for tapper container")
return errors.New(fmt.Sprintf("invalid memory request for %s container", tapperPodName))
}
agentResourceLimits := core.ResourceList{
"cpu": cpuLimit,
"cpu": cpuLimit,
"memory": memLimit,
}
agentResourceRequests := core.ResourceList{
"cpu": cpuRequests,
"cpu": cpuRequests,
"memory": memRequests,
}
agentResources := applyconfcore.ResourceRequirements().WithRequests(agentResourceRequests).WithLimits(agentResourceLimits)
@ -425,7 +656,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
podSpec.WithHostNetwork(true)
podSpec.WithDNSPolicy(core.DNSClusterFirstWithHostNet)
podSpec.WithTerminationGracePeriodSeconds(0)
if linkServiceAccount {
if serviceAccountName != "" {
podSpec.WithServiceAccountName(serviceAccountName)
}
podSpec.WithContainers(agentContainer)
@ -446,18 +677,55 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
return err
}
func (provider *Provider) GetAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespace string) ([]core.Pod, error) {
pods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
if err != nil {
return nil, err
func (provider *Provider) ListAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespaces []string) ([]core.Pod, error) {
var pods []core.Pod
for _, namespace := range namespaces {
namespacePods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("failed to get pods in ns: [%s], %w", namespace, err)
}
pods = append(pods, namespacePods.Items...)
}
matchingPods := make([]core.Pod, 0)
for _, pod := range pods.Items {
for _, pod := range pods {
if regex.MatchString(pod.Name) {
matchingPods = append(matchingPods, pod)
}
}
return matchingPods, err
return matchingPods, nil
}
func (provider *Provider) ListAllRunningPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespaces []string) ([]core.Pod, error) {
pods, err := provider.ListAllPodsMatchingRegex(ctx, regex, namespaces)
if err != nil {
return nil, err
}
matchingPods := make([]core.Pod, 0)
for _, pod := range pods {
if isPodRunning(&pod) {
matchingPods = append(matchingPods, pod)
}
}
return matchingPods, nil
}
func (provider *Provider) GetPodLogs(namespace string, podName string, ctx context.Context) (string, error) {
podLogOpts := core.PodLogOptions{}
req := provider.clientSet.CoreV1().Pods(namespace).GetLogs(podName, &podLogOpts)
podLogs, err := req.Stream(ctx)
if err != nil {
return "", fmt.Errorf("error opening log stream on ns: %s, pod: %s, %w", namespace, podName, err)
}
defer podLogs.Close()
buf := new(bytes.Buffer)
if _, err = io.Copy(buf, podLogs); err != nil {
return "", fmt.Errorf("error copy information from podLogs to buf, ns: %s, pod: %s, %w", namespace, podName, err)
}
str := buf.String()
return str, nil
}
func getClientSet(config *restclient.Config) *kubernetes.Clientset {
@ -469,11 +737,16 @@ func getClientSet(config *restclient.Config) *kubernetes.Clientset {
}
func loadKubernetesConfiguration(kubeConfigPath string) clientcmd.ClientConfig {
if kubeConfigPath == "" {
kubeConfigPath = os.Getenv("KUBECONFIG")
}
if kubeConfigPath == "" {
home := homedir.HomeDir()
kubeConfigPath = filepath.Join(home, ".kube", "config")
}
mizu.Log.Debugf("Using kube config %s", kubeConfigPath)
configPathList := filepath.SplitList(kubeConfigPath)
configLoadingRules := &clientcmd.ClientConfigLoadingRules{}
if len(configPathList) <= 1 {
@ -489,3 +762,7 @@ func loadKubernetesConfiguration(kubeConfigPath string) clientcmd.ClientConfig {
},
)
}
func isPodRunning(pod *core.Pod) bool {
return pod.Status.Phase == core.PodRunning
}

10
cli/kubernetes/proxy.go
View File

@ -2,6 +2,7 @@ package kubernetes
import (
"fmt"
"github.com/up9inc/mizu/cli/mizu"
"k8s.io/kubectl/pkg/proxy"
"net"
"net/http"
@ -13,6 +14,7 @@ const k8sProxyApiPrefix = "/"
const mizuServicePort = 80
func StartProxy(kubernetesProvider *Provider, mizuPort uint16, mizuNamespace string, mizuServiceName string) error {
mizu.Log.Debugf("Starting proxy. namespace: [%v], service name: [%s], port: [%v]", mizuNamespace, mizuServiceName, mizuPort)
filter := &proxy.FilterServer{
AcceptPaths: proxy.MakeRegexpArrayOrDie(proxy.DefaultPathAcceptRE),
RejectPaths: proxy.MakeRegexpArrayOrDie(proxy.DefaultPathRejectRE),
@ -40,24 +42,24 @@ func StartProxy(kubernetesProvider *Provider, mizuPort uint16, mizuNamespace str
return server.Serve(l)
}
func getMizuCollectorProxiedHostAndPath(mizuNamespace string, mizuServiceName string) string {
func getMizuApiServerProxiedHostAndPath(mizuNamespace string, mizuServiceName string) string {
return fmt.Sprintf("/api/v1/namespaces/%s/services/%s:%d/proxy/", mizuNamespace, mizuServiceName, mizuServicePort)
}
func GetMizuCollectorProxiedHostAndPath(mizuPort uint16) string {
func GetMizuApiServerProxiedHostAndPath(mizuPort uint16) string {
return fmt.Sprintf("localhost:%d/mizu", mizuPort)
}
func getRerouteHttpHandlerMizuAPI(proxyHandler http.Handler, mizuNamespace string, mizuServiceName string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r.URL.Path = strings.Replace(r.URL.Path, "/mizu/", getMizuCollectorProxiedHostAndPath(mizuNamespace, mizuServiceName), 1)
r.URL.Path = strings.Replace(r.URL.Path, "/mizu/", getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName), 1)
proxyHandler.ServeHTTP(w, r)
})
}
func getRerouteHttpHandlerMizuStatic(proxyHandler http.Handler, mizuNamespace string, mizuServiceName string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r.URL.Path = strings.Replace(r.URL.Path, "/static/", fmt.Sprintf("%s/static/", getMizuCollectorProxiedHostAndPath(mizuNamespace, mizuServiceName)), 1)
r.URL.Path = strings.Replace(r.URL.Path, "/static/", fmt.Sprintf("%s/static/", getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName)), 1)
proxyHandler.ServeHTTP(w, r)
})
}

75
cli/kubernetes/watch.go
View File

@ -4,49 +4,64 @@ import (
"context"
"errors"
"regexp"
"sync"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/watch"
)
func FilteredWatch(ctx context.Context, watcher watch.Interface, podFilter *regexp.Regexp) (chan *corev1.Pod, chan *corev1.Pod, chan *corev1.Pod, chan error) {
func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetNamespaces []string, podFilter *regexp.Regexp) (chan *corev1.Pod, chan *corev1.Pod, chan *corev1.Pod, chan error) {
addedChan := make(chan *corev1.Pod)
modifiedChan := make(chan *corev1.Pod)
removedChan := make(chan *corev1.Pod)
errorChan := make(chan error)
go func() {
for {
select {
case e := <-watcher.ResultChan():
if e.Object == nil {
errorChan <- errors.New("kubernetes pod watch failed")
var wg sync.WaitGroup
for _, targetNamespace := range targetNamespaces {
wg.Add(1)
go func(targetNamespace string) {
defer wg.Done()
watcher := kubernetesProvider.GetPodWatcher(ctx, targetNamespace)
for {
select {
case e := <-watcher.ResultChan():
if e.Object == nil {
errorChan <- errors.New("kubernetes pod watch failed")
return
}
pod := e.Object.(*corev1.Pod)
if !podFilter.MatchString(pod.Name) {
continue
}
switch e.Type {
case watch.Added:
addedChan <- pod
case watch.Modified:
modifiedChan <- pod
case watch.Deleted:
removedChan <- pod
}
case <-ctx.Done():
watcher.Stop()
return
}
pod := e.Object.(*corev1.Pod)
if !podFilter.MatchString(pod.Name) {
continue
}
switch e.Type {
case watch.Added:
addedChan <- pod
case watch.Modified:
modifiedChan <- pod
case watch.Deleted:
removedChan <- pod
}
case <-ctx.Done():
watcher.Stop()
close(addedChan)
close(modifiedChan)
close(removedChan)
close(errorChan)
return
}
}
}(targetNamespace)
}
go func() {
<-ctx.Done()
wg.Wait()
close(addedChan)
close(modifiedChan)
close(removedChan)
close(errorChan)
}()
return addedChan, modifiedChan, removedChan, errorChan

7
cli/mizu.go
View File

@ -1,7 +1,10 @@
package main
import "github.com/up9inc/mizu/cli/cmd"
import (
"github.com/up9inc/mizu/cli/cmd"
"github.com/up9inc/mizu/cli/goUtils"
)
func main() {
cmd.Execute()
goUtils.HandleExcWrapper(cmd.Execute)
}

279
cli/mizu/config.go Normal file
View File

@ -0,0 +1,279 @@
package mizu
import (
"errors"
"fmt"
"io/ioutil"
"os"
"path"
"reflect"
"strconv"
"strings"
"github.com/creasty/defaults"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/up9inc/mizu/cli/mizu/configStructs"
"github.com/up9inc/mizu/cli/uiUtils"
"gopkg.in/yaml.v3"
)
const (
Separator = "="
SetCommandName = "set"
)
var allowedSetFlags = []string{
AgentImageConfigName,
MizuResourcesNamespaceConfigName,
TelemetryConfigName,
DumpLogsConfigName,
KubeConfigPathName,
configStructs.AnalysisDestinationTapName,
configStructs.SleepIntervalSecTapName,
}
var Config = ConfigStruct{}
func (config *ConfigStruct) Validate() error {
if config.IsNsRestrictedMode() {
if config.Tap.AllNamespaces || len(config.Tap.Namespaces) != 1 || config.Tap.Namespaces[0] != config.MizuResourcesNamespace {
return fmt.Errorf("Not supported mode. Mizu can't resolve IPs in other namespaces when running in namespace restricted mode.\n"+
"You can use the same namespace for --%s and --%s", configStructs.NamespacesTapName, MizuResourcesNamespaceConfigName)
}
}
return nil
}
func InitConfig(cmd *cobra.Command) error {
if err := defaults.Set(&Config); err != nil {
return err
}
if err := mergeConfigFile(); err != nil {
return fmt.Errorf("invalid config %w\n"+
"you can regenerate the file using `mizu config -r` or just remove it %v", err, GetConfigFilePath())
}
cmd.Flags().Visit(initFlag)
finalConfigPrettified, _ := uiUtils.PrettyJson(Config)
Log.Debugf("Init config finished\n Final config: %v", finalConfigPrettified)
return nil
}
func GetConfigWithDefaults() (string, error) {
defaultConf := ConfigStruct{}
if err := defaults.Set(&defaultConf); err != nil {
return "", err
}
return uiUtils.PrettyYaml(defaultConf)
}
func GetConfigFilePath() string {
return path.Join(GetMizuFolderPath(), "config.yaml")
}
func mergeConfigFile() error {
reader, openErr := os.Open(GetConfigFilePath())
if openErr != nil {
return nil
}
buf, readErr := ioutil.ReadAll(reader)
if readErr != nil {
return readErr
}
if err := yaml.Unmarshal(buf, &Config); err != nil {
return err
}
Log.Debugf("Found config file, merged to default options")
return nil
}
func initFlag(f *pflag.Flag) {
configElem := reflect.ValueOf(&Config).Elem()
sliceValue, isSliceValue := f.Value.(pflag.SliceValue)
if !isSliceValue {
mergeFlagValue(configElem, f.Name, f.Value.String())
return
}
if f.Name == SetCommandName {
mergeSetFlag(sliceValue.GetSlice())
return
}
mergeFlagValues(configElem, f.Name, sliceValue.GetSlice())
}
func mergeSetFlag(setValues []string) {
configElem := reflect.ValueOf(&Config).Elem()
for _, setValue := range setValues {
if !strings.Contains(setValue, Separator) {
Log.Warningf(uiUtils.Warning, fmt.Sprintf("Ignoring set argument %s (set argument format: <flag name>=<flag value>)", setValue))
}
split := strings.SplitN(setValue, Separator, 2)
if len(split) != 2 {
Log.Warningf(uiUtils.Warning, fmt.Sprintf("Ignoring set argument %s (set argument format: <flag name>=<flag value>)", setValue))
}
argumentKey, argumentValue := split[0], split[1]
if !Contains(allowedSetFlags, argumentKey) {
Log.Warningf(uiUtils.Warning, fmt.Sprintf("Ignoring set argument %s, flag name must be one of the following: \"%s\"", setValue, strings.Join(allowedSetFlags, "\", \"")))
}
mergeFlagValue(configElem, argumentKey, argumentValue)
}
}
func mergeFlagValue(currentElem reflect.Value, flagKey string, flagValue string) {
for i := 0; i < currentElem.NumField(); i++ {
currentField := currentElem.Type().Field(i)
currentFieldByName := currentElem.FieldByName(currentField.Name)
if currentField.Type.Kind() == reflect.Struct {
mergeFlagValue(currentFieldByName, flagKey, flagValue)
continue
}
if currentField.Tag.Get("yaml") != flagKey {
continue
}
flagValueKind := currentField.Type.Kind()
parsedValue, err := getParsedValue(flagValueKind, flagValue)
if err != nil {
Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for flag name %s, expected %s", flagValue, flagKey, flagValueKind))
return
}
currentFieldByName.Set(parsedValue)
}
}
func mergeFlagValues(currentElem reflect.Value, flagKey string, flagValues []string) {
for i := 0; i < currentElem.NumField(); i++ {
currentField := currentElem.Type().Field(i)
currentFieldByName := currentElem.FieldByName(currentField.Name)
if currentField.Type.Kind() == reflect.Struct {
mergeFlagValues(currentFieldByName, flagKey, flagValues)
continue
}
if currentField.Tag.Get("yaml") != flagKey {
continue
}
flagValueKind := currentField.Type.Elem().Kind()
parsedValues := reflect.MakeSlice(reflect.SliceOf(currentField.Type.Elem()), 0, 0)
for _, flagValue := range flagValues {
parsedValue, err := getParsedValue(flagValueKind, flagValue)
if err != nil {
Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for flag name %s, expected %s", flagValue, flagKey, flagValueKind))
return
}
parsedValues = reflect.Append(parsedValues, parsedValue)
}
currentFieldByName.Set(parsedValues)
}
}
func getParsedValue(kind reflect.Kind, value string) (reflect.Value, error) {
switch kind {
case reflect.String:
return reflect.ValueOf(value), nil
case reflect.Bool:
boolArgumentValue, err := strconv.ParseBool(value)
if err != nil {
break
}
return reflect.ValueOf(boolArgumentValue), nil
case reflect.Int:
intArgumentValue, err := strconv.ParseInt(value, 10, 64)
if err != nil {
break
}
return reflect.ValueOf(int(intArgumentValue)), nil
case reflect.Int8:
intArgumentValue, err := strconv.ParseInt(value, 10, 8)
if err != nil {
break
}
return reflect.ValueOf(int8(intArgumentValue)), nil
case reflect.Int16:
intArgumentValue, err := strconv.ParseInt(value, 10, 16)
if err != nil {
break
}
return reflect.ValueOf(int16(intArgumentValue)), nil
case reflect.Int32:
intArgumentValue, err := strconv.ParseInt(value, 10, 32)
if err != nil {
break
}
return reflect.ValueOf(int32(intArgumentValue)), nil
case reflect.Int64:
intArgumentValue, err := strconv.ParseInt(value, 10, 64)
if err != nil {
break
}
return reflect.ValueOf(intArgumentValue), nil
case reflect.Uint:
uintArgumentValue, err := strconv.ParseUint(value, 10, 64)
if err != nil {
break
}
return reflect.ValueOf(uint(uintArgumentValue)), nil
case reflect.Uint8:
uintArgumentValue, err := strconv.ParseUint(value, 10, 8)
if err != nil {
break
}
return reflect.ValueOf(uint8(uintArgumentValue)), nil
case reflect.Uint16:
uintArgumentValue, err := strconv.ParseUint(value, 10, 16)
if err != nil {
break
}
return reflect.ValueOf(uint16(uintArgumentValue)), nil
case reflect.Uint32:
uintArgumentValue, err := strconv.ParseUint(value, 10, 32)
if err != nil {
break
}
return reflect.ValueOf(uint32(uintArgumentValue)), nil
case reflect.Uint64:
uintArgumentValue, err := strconv.ParseUint(value, 10, 64)
if err != nil {
break
}
return reflect.ValueOf(uintArgumentValue), nil
}
return reflect.ValueOf(nil), errors.New("value to parse does not match type")
}

35
cli/mizu/configStruct.go Normal file
View File

@ -0,0 +1,35 @@
package mizu
import (
"fmt"
"github.com/up9inc/mizu/cli/mizu/configStructs"
)
const (
AgentImageConfigName = "agent-image"
MizuResourcesNamespaceConfigName = "mizu-resources-namespace"
TelemetryConfigName = "telemetry"
DumpLogsConfigName = "dump-logs"
KubeConfigPathName = "kube-config-path"
)
type ConfigStruct struct {
Tap configStructs.TapConfig `yaml:"tap"`
Fetch configStructs.FetchConfig `yaml:"fetch"`
Version configStructs.VersionConfig `yaml:"version"`
View configStructs.ViewConfig `yaml:"view"`
AgentImage string `yaml:"agent-image"`
MizuResourcesNamespace string `yaml:"mizu-resources-namespace" default:"mizu"`
Telemetry bool `yaml:"telemetry" default:"true"`
DumpLogs bool `yaml:"dump-logs" default:"false"`
KubeConfigPath string `yaml:"kube-config-path" default:""`
}
func (config *ConfigStruct) SetDefaults() {
config.AgentImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", Branch, SemVer)
}
func (config *ConfigStruct) IsNsRestrictedMode() bool {
return config.MizuResourcesNamespace != "mizu" // Notice "mizu" string must match the default MizuResourcesNamespace
}

15
cli/mizu/configStructs/fetchConfig.go Normal file
View File

@ -0,0 +1,15 @@
package configStructs
const (
DirectoryFetchName = "directory"
FromTimestampFetchName = "from"
ToTimestampFetchName = "to"
GuiPortFetchName = "gui-port"
)
type FetchConfig struct {
Directory string `yaml:"directory" default:"."`
FromTimestamp int `yaml:"from" default:"0"`
ToTimestamp int `yaml:"to" default:"0"`
GuiPort uint16 `yaml:"gui-port" default:"8899"`
}

81
cli/mizu/configStructs/tapConfig.go Normal file
View File

@ -0,0 +1,81 @@
package configStructs
import (
"errors"
"fmt"
"regexp"
"strings"
"github.com/up9inc/mizu/shared/units"
)
const (
AnalysisDestinationTapName = "dest"
SleepIntervalSecTapName = "upload-interval"
GuiPortTapName = "gui-port"
NamespacesTapName = "namespaces"
AnalysisTapName = "analysis"
AllNamespacesTapName = "all-namespaces"
PlainTextFilterRegexesTapName = "regex-masking"
HideHealthChecksTapName = "hide-healthchecks"
DisableRedactionTapName = "no-redact"
HumanMaxEntriesDBSizeTapName = "max-entries-db-size"
DirectionTapName = "direction"
DryRunTapName = "dry-run"
EnforcePolicyFile = "test-rules"
)
type TapConfig struct {
AnalysisDestination string `yaml:"dest" default:"up9.app"`
SleepIntervalSec int `yaml:"upload-interval" default:"10"`
PodRegexStr string `yaml:"regex" default:".*"`
GuiPort uint16 `yaml:"gui-port" default:"8899"`
Namespaces []string `yaml:"namespaces"`
Analysis bool `yaml:"analysis" default:"false"`
AllNamespaces bool `yaml:"all-namespaces" default:"false"`
PlainTextFilterRegexes []string `yaml:"regex-masking"`
HideHealthChecks bool `yaml:"hide-healthchecks" default:"false"`
DisableRedaction bool `yaml:"no-redact" default:"false"`
HumanMaxEntriesDBSize string `yaml:"max-entries-db-size" default:"200MB"`
Direction string `yaml:"direction" default:"in"`
DryRun bool `yaml:"dry-run" default:"false"`
EnforcePolicyFile string `yaml:"test-rules"`
}
func (config *TapConfig) PodRegex() *regexp.Regexp {
podRegex, _ := regexp.Compile(config.PodRegexStr)
return podRegex
}
func (config *TapConfig) TapOutgoing() bool {
directionLowerCase := strings.ToLower(config.Direction)
if directionLowerCase == "any" {
return true
}
return false
}
func (config *TapConfig) MaxEntriesDBSizeBytes() int64 {
maxEntriesDBSizeBytes, _ := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
return maxEntriesDBSizeBytes
}
func (config *TapConfig) Validate() error {
_, compileErr := regexp.Compile(config.PodRegexStr)
if compileErr != nil {
return errors.New(fmt.Sprintf("%s is not a valid regex %s", config.PodRegexStr, compileErr))
}
_, parseHumanDataSizeErr := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
if parseHumanDataSizeErr != nil {
return errors.New(fmt.Sprintf("Could not parse --%s value %s", HumanMaxEntriesDBSizeTapName, config.HumanMaxEntriesDBSize))
}
directionLowerCase := strings.ToLower(config.Direction)
if directionLowerCase != "any" && directionLowerCase != "in" {
return errors.New(fmt.Sprintf("%s is not a valid value for flag --%s. Acceptable values are in/any.", config.Direction, DirectionTapName))
}
return nil
}

9
cli/mizu/configStructs/versionConfig.go Normal file
View File

@ -0,0 +1,9 @@
package configStructs
const (
DebugInfoVersionName = "debug"
)
type VersionConfig struct {
DebugInfo bool `yaml:"debug" default:"false"`
}

11
cli/mizu/configStructs/viewConfig.go Normal file
View File

@ -0,0 +1,11 @@
package configStructs
const (
GuiPortViewName = "gui-port"
KubeConfigPathViewName = "kube-config"
)
type ViewConfig struct {
GuiPort uint16 `yaml:"gui-port" default:"8899"`
KubeConfigPath string `yaml:"kube-config"`
}

38
cli/mizu/consts.go
View File

@ -1,5 +1,10 @@
package mizu
import (
"os"
"path"
)
var (
SemVer = "0.0.1"
Branch = "develop"
@ -9,20 +14,23 @@ var (
)
const (
ResourcesNamespace = "default"
TapperDaemonSetName = "mizu-tapper-daemon-set"
AggregatorPodName = "mizu-collector"
TapperPodName = "mizu-tapper"
K8sAllNamespaces = ""
MizuResourcesPrefix = "mizu-"
ApiServerPodName = MizuResourcesPrefix + "api-server"
ClusterRoleBindingName = MizuResourcesPrefix + "cluster-role-binding"
ClusterRoleName = MizuResourcesPrefix + "cluster-role"
K8sAllNamespaces = ""
RoleBindingName = MizuResourcesPrefix + "role-binding"
RoleName = MizuResourcesPrefix + "role"
ServiceAccountName = MizuResourcesPrefix + "service-account"
TapperDaemonSetName = MizuResourcesPrefix + "tapper-daemon-set"
TapperPodName = MizuResourcesPrefix + "tapper"
ConfigMapName = MizuResourcesPrefix + "policy"
)
const (
Black = "\033[1;30m%s\033[0m"
Red = "\033[1;31m%s\033[0m"
Green = "\033[1;32m%s\033[0m"
Yellow = "\033[1;33m%s\033[0m"
Purple = "\033[1;34m%s\033[0m"
Magenta = "\033[1;35m%s\033[0m"
Teal = "\033[1;36m%s\033[0m"
White = "\033[1;37m%s\033[0m"
)
func GetMizuFolderPath() string {
home, homeDirErr := os.UserHomeDir()
if homeDirErr != nil {
return ""
}
return path.Join(home, ".mizu")
}

38
cli/mizu/logger.go Normal file
View File

@ -0,0 +1,38 @@
package mizu
import (
"github.com/op/go-logging"
"os"
"path"
)
var Log = logging.MustGetLogger("mizu_cli")
var format = logging.MustStringFormatter(
`%{time} %{level:.5s} ▶ %{pid} %{shortfile} %{shortfunc} ▶ %{message}`,
)
func GetLogFilePath() string {
return path.Join(GetMizuFolderPath(), "mizu_cli.log")
}
func InitLogger() {
logPath := GetLogFilePath()
f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
Log.Infof("Failed to open mizu log file: %v, err %v", logPath, err)
}
fileLog := logging.NewLogBackend(f, "", 0)
consoleLog := logging.NewLogBackend(os.Stderr, "", 0)
backend2Formatter := logging.NewBackendFormatter(fileLog, format)
backend1Leveled := logging.AddModuleLevel(consoleLog)
backend1Leveled.SetLevel(logging.INFO, "")
logging.SetBackend(backend1Leveled, backend2Formatter)
Log.Debugf("\n\n\n")
Log.Debugf("Running mizu version %v", SemVer)
}

11
cli/mizu/sliceUtils.go Normal file
View File

@ -0,0 +1,11 @@
package mizu
func Contains(slice []string, containsValue string) bool {
for _, sliceValue := range slice {
if sliceValue == containsValue {
return true
}
}
return false
}

36
cli/mizu/telemetry.go Normal file
View File

@ -0,0 +1,36 @@
package mizu
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
)
const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
func ReportRun(cmd string, args interface{}) {
if !Config.Telemetry {
Log.Debugf("not reporting due to config value")
return
}
argsBytes, _ := json.Marshal(args)
argsMap := map[string]string{
"telemetry_type": "execution",
"cmd": cmd,
"args": string(argsBytes),
"component": "mizu_cli",
"BuildTimestamp": BuildTimestamp,
"Branch": Branch,
"version": SemVer}
argsMap["message"] = fmt.Sprintf("mizu %v - %v", argsMap["cmd"], string(argsBytes))
jsonValue, _ := json.Marshal(argsMap)
if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
Log.Debugf("error sending telemetry err: %v, response %v", err, resp)
} else {
Log.Debugf("Successfully reported telemetry")
}
}

55
cli/mizu/versionCheck.go
View File

@ -1,14 +1,19 @@
package mizu
import (
"context"
"encoding/json"
"fmt"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/semver"
"io/ioutil"
"net/http"
"net/url"
)
"time"
"github.com/google/go-github/v37/github"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/semver"
)
func getApiVersion(port uint16) (string, error) {
versionUrl, _ := url.Parse(fmt.Sprintf("http://localhost:%d/mizu/metadata/version", port))
@ -41,6 +46,48 @@ func CheckVersionCompatibility(port uint16) (bool, error) {
return true, nil
}
fmt.Printf(Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)\n", SemVer, apiSemVer))
Log.Errorf(uiUtils.Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)", SemVer, apiSemVer))
return false, nil
}
func CheckNewerVersion() {
Log.Debugf("Checking for newer version...")
start := time.Now()
client := github.NewClient(nil)
latestRelease, _, err := client.Repositories.GetLatestRelease(context.Background(), "up9inc", "mizu")
if err != nil {
Log.Debugf("[ERROR] Failed to get latest release")
return
}
versionFileUrl := ""
for _, asset := range latestRelease.Assets {
if *asset.Name == "version.txt" {
versionFileUrl = *asset.BrowserDownloadURL
break
}
}
if versionFileUrl == "" {
Log.Debugf("[ERROR] Version file not found in the latest release")
return
}
res, err := http.Get(versionFileUrl)
if err != nil {
Log.Debugf("[ERROR] Failed to get the version file %v", err)
return
}
data, err := ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
Log.Debugf("[ERROR] Failed to read the version file -> %v", err)
return
}
gitHubVersion := string(data)
gitHubVersion = gitHubVersion[:len(gitHubVersion)-1]
Log.Debugf("Finished version validation, took %v", time.Since(start))
if SemVer < gitHubVersion {
Log.Infof(uiUtils.Yellow, fmt.Sprintf("Update available! %v -> %v (%v)", SemVer, gitHubVersion, *latestRelease.HTMLURL))
}
}

15
cli/uiUtils/colors.go Normal file
View File

@ -0,0 +1,15 @@
package uiUtils
const (
Black = "\033[1;30m%s\033[0m"
Red = "\033[1;31m%s\033[0m"
Green = "\033[1;32m%s\033[0m"
Yellow = "\033[1;33m%s\033[0m"
Purple = "\033[1;34m%s\033[0m"
Magenta = "\033[1;35m%s\033[0m"
Teal = "\033[1;36m%s\033[0m"
White = "\033[1;37m%s\033[0m"
Error = Red
Warning = Yellow
)

3
cli/uiUtils/confirmation.go
View File

@ -3,7 +3,6 @@ package uiUtils
import (
"bufio"
"fmt"
"github.com/up9inc/mizu/cli/mizu"
"log"
"os"
"strings"
@ -12,7 +11,7 @@ import (
func AskForConfirmation(s string) bool {
reader := bufio.NewReader(os.Stdin)
fmt.Printf(mizu.Magenta, s)
fmt.Printf(Magenta, s)
response, err := reader.ReadString('\n')
if err != nil {

36
cli/uiUtils/prettyString.go Normal file
View File

@ -0,0 +1,36 @@
package uiUtils
import (
"bytes"
"encoding/json"
"gopkg.in/yaml.v3"
)
const (
empty = ""
tab = "\t"
)
func PrettyJson(data interface{}) (string, error) {
buffer := new(bytes.Buffer)
encoder := json.NewEncoder(buffer)
encoder.SetIndent(empty, tab)
err := encoder.Encode(data)
if err != nil {
return empty, err
}
return buffer.String(), nil
}
func PrettyYaml(data interface{}) (string, error) {
buffer := new(bytes.Buffer)
encoder := yaml.NewEncoder(buffer)
encoder.SetIndent(0)
err := encoder.Encode(data)
if err != nil {
return empty, err
}
return buffer.String(), nil
}

20
debug.Dockerfile
View File

@ -1,4 +1,4 @@
# creates image in which mizu api is remotely debuggable using delve
# creates image in which mizu agent is remotely debuggable using delve
FROM node:14-slim AS site-build
WORKDIR /app/ui-build
@ -14,18 +14,21 @@ ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
RUN apk add libpcap-dev gcc g++ make
# Move to api working directory (/api-build).
WORKDIR /app/api-build
# Move to agent working directory (/agent-build).
WORKDIR /app/agent-build
COPY api/go.mod api/go.sum ./
COPY agent/go.mod agent/go.sum ./
COPY shared/go.mod shared/go.mod ../shared/
COPY tap/go.mod tap/go.mod ../tap/
RUN go mod download
# cheap trick to make the build faster (As long as go.mod wasn't changes)
RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
# Copy and build api code
# Copy and build agent code
COPY shared ../shared
COPY api .
COPY tap ../tap
COPY agent .
RUN go build -gcflags="all=-N -l" -o mizuagent .
@ -35,10 +38,11 @@ RUN apk add bash libpcap-dev tcpdump
WORKDIR /app
# Copy binary and config files from /build to root folder of scratch container.
COPY --from=builder ["/app/api-build/mizuagent", "."]
COPY --from=builder ["/app/agent-build/mizuagent", "."]
COPY --from=site-build ["/app/ui-build/build", "site"]
# install remote debugging tool
RUN go get github.com/go-delve/delve/cmd/dlv
CMD ["sh", "-c", "dlv --headless=true --listen=:2345 --log --api-version=2 --accept-multiclient exec ./mizuagent -- --aggregator"]
ENTRYPOINT "/app/mizuagent"
#CMD ["sh", "-c", "dlv --headless=true --listen=:2345 --log --api-version=2 --accept-multiclient exec ./mizuagent -- --api-server"]

35
examples/roles/permissions-all-namespaces-without-ip-resolution.yaml Normal file
View File

@ -0,0 +1,35 @@
# This example shows the roles required for a user to be able to use Mizu in all namespaces with IP resolution disabled.
# (Traffic will be recorded, but Mizu will not translate IP addresses to names)
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mizu-runner-clusterrole
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["services"]
verbs: ["create", "delete"]
- apiGroups: ["apps"]
resources: ["daemonsets"]
verbs: ["create", "patch", "delete"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["services/proxy"]
verbs: ["get"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mizu-runner-clusterrolebindings
subjects:
- kind: User
name: user1
apiGroup: rbac.authorization.k8s.io
roleRef:
kind: ClusterRole
name: mizu-runner-clusterrole
apiGroup: rbac.authorization.k8s.io

58
examples/roles/permissions-all-namespaces.yaml Normal file
View File

@ -0,0 +1,58 @@
# This example shows the roles required for a user to be able to use Mizu in all namespaces.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mizu-runner-clusterrole
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["services"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: ["apps"]
resources: ["daemonsets"]
verbs: ["create", "patch", "delete"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["services/proxy"]
verbs: ["get"]
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["get", "create", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["clusterroles"]
verbs: ["get", "create", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["clusterrolebindings"]
verbs: ["get", "create", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["roles"]
verbs: ["get", "create", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["rolebindings"]
verbs: ["get", "create", "delete"]
- apiGroups: ["apps", "extensions"]
resources: ["pods"]
verbs: ["get", "list", "watch"]
- apiGroups: ["apps", "extensions"]
resources: ["services"]
verbs: ["get", "list", "watch"]
- apiGroups: ["", "apps", "extensions"]
resources: ["endpoints"]
verbs: ["get", "list", "watch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mizu-runner-clusterrolebindings
subjects:
- kind: User
name: user1
apiGroup: rbac.authorization.k8s.io
roleRef:
kind: ClusterRole
name: mizu-runner-clusterrole
apiGroup: rbac.authorization.k8s.io

54
examples/roles/permissions-ns-with-validation.yaml Normal file
View File

@ -0,0 +1,54 @@
# This example shows the roles required for a user to be able to use Mizu in a single namespace.
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mizu-runner-role
namespace: user1
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["services"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: ["apps"]
resources: ["daemonsets"]
verbs: ["get", "create", "patch", "delete"]
- apiGroups: [""]
resources: ["services/proxy"]
verbs: ["get"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "create", "delete"]
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["get", "create", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["roles"]
verbs: ["get", "create", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["rolebindings"]
verbs: ["get", "create", "delete"]
- apiGroups: ["apps", "extensions"]
resources: ["pods"]
verbs: ["get", "list", "watch"]
- apiGroups: ["apps", "extensions"]
resources: ["services"]
verbs: ["get", "list", "watch"]
- apiGroups: ["", "apps", "extensions"]
resources: ["endpoints"]
verbs: ["get", "list", "watch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mizu-runner-rolebindings
namespace: user1
subjects:
- kind: User
name: user1
apiGroup: rbac.authorization.k8s.io
roleRef:
kind: Role
name: mizu-runner-role
apiGroup: rbac.authorization.k8s.io

33
examples/roles/permissions-ns-without-ip-resolution.yaml Normal file
View File

@ -0,0 +1,33 @@
# This example shows the roles required for a user to be able to use Mizu in a single namespace with IP resolution disabled.
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mizu-runner-role
namespace: user1
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["services"]
verbs: ["get", "create", "delete"]
- apiGroups: ["apps"]
resources: ["daemonsets"]
verbs: ["get", "create", "patch", "delete"]
- apiGroups: [""]
resources: ["services/proxy"]
verbs: ["get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mizu-runner-rolebindings
namespace: user1
subjects:
- kind: User
name: user1
apiGroup: rbac.authorization.k8s.io
roleRef:
kind: Role
name: mizu-runner-role
apiGroup: rbac.authorization.k8s.io

51
examples/roles/permissions-ns.yaml Normal file
View File

@ -0,0 +1,51 @@
# This example shows the roles required for a user to be able to use Mizu in a single namespace.
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mizu-runner-role
namespace: user1
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["services"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: ["apps"]
resources: ["daemonsets"]
verbs: ["get", "create", "patch", "delete"]
- apiGroups: [""]
resources: ["services/proxy"]
verbs: ["get"]
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["get", "create", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["roles"]
verbs: ["get", "create", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["rolebindings"]
verbs: ["get", "create", "delete"]
- apiGroups: ["apps", "extensions"]
resources: ["pods"]
verbs: ["get", "list", "watch"]
- apiGroups: ["apps", "extensions"]
resources: ["services"]
verbs: ["get", "list", "watch"]
- apiGroups: ["", "apps", "extensions"]
resources: ["endpoints"]
verbs: ["get", "list", "watch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mizu-runner-rolebindings
namespace: user1
subjects:
- kind: User
name: user1
apiGroup: rbac.authorization.k8s.io
roleRef:
kind: Role
name: mizu-runner-role
apiGroup: rbac.authorization.k8s.io

4
shared/consts.go
View File

@ -5,5 +5,7 @@ const (
HostModeEnvVar = "HOST_MODE"
NodeNameEnvVar = "NODE_NAME"
TappedAddressesPerNodeDictEnvVar = "TAPPED_ADDRESSES_PER_HOST"
MaxEntriesDBSizeByteSEnvVar = "MAX_ENTRIES_DB_BYTES"
MaxEntriesDBSizeBytesEnvVar = "MAX_ENTRIES_DB_BYTES"
RulePolicyPath = "/app/enforce-policy/"
RulePolicyFileName = "enforce-policy.yaml"
)

24
shared/debounce/debounce.go
View File

@ -1,6 +1,7 @@
package debounce
import (
"fmt"
"time"
)
@ -13,9 +14,10 @@ func NewDebouncer(timeout time.Duration, callback func()) *Debouncer {
type Debouncer struct {
callback func()
running bool
timeout time.Duration
timer *time.Timer
running bool
canceled bool
timeout time.Duration
timer *time.Timer
}
func (d *Debouncer) setTimeout(timeout time.Duration) {
@ -25,18 +27,28 @@ func (d *Debouncer) setTimeout(timeout time.Duration) {
func (d *Debouncer) setCallback(callback func()) {
callbackWrapped := func() {
callback()
if !d.canceled {
callback()
}
d.running = false
}
d.callback = callbackWrapped
}
func (d *Debouncer) SetOn() {
func (d *Debouncer) Cancel() {
d.canceled = true
}
func (d *Debouncer) SetOn() error {
if d.canceled {
return fmt.Errorf("debouncer cancelled")
}
if d.running == true {
return
return nil
}
d.running = true
d.timer = time.AfterFunc(d.timeout, d.callback)
return nil
}

Some files were not shown because too many files have changed in this diff Show More