diff --git a/.github/workflows/acceptance_tests.yml b/.github/workflows/acceptance_tests.yml index a23f3c445..5486bb755 100644 --- a/.github/workflows/acceptance_tests.yml +++ b/.github/workflows/acceptance_tests.yml @@ -30,3 +30,15 @@ jobs: - name: Test run: make acceptance-test + + - name: Slack notification on failure + uses: ravsamhq/notify-slack-action@v1 + if: always() + with: + status: ${{ job.status }} + notification_title: 'Mizu {workflow} has {status_message}' + message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit: <{commit_url}|{commit_sha}>' + footer: 'Linked Repo <{repo_url}|{repo}>' + notify_when: 'failure' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/inactive-issues-close.yaml b/.github/workflows/inactive-issues-close.yaml new file mode 100644 index 000000000..e8509276f --- /dev/null +++ b/.github/workflows/inactive-issues-close.yaml @@ -0,0 +1,22 @@ +name: Close inactive issues +on: + schedule: + - cron: "0 0 * * *" + +jobs: + close-issues: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v3 + with: + days-before-issue-stale: 30 + days-before-issue-close: 14 + stale-issue-label: "stale" + stale-issue-message: "This issue is stale because it has been open for 30 days with no activity." + close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale." + days-before-pr-stale: -1 + days-before-pr-close: -1 + repo-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/security_validation.yml b/.github/workflows/security_validation.yml deleted file mode 100644 index 2808638a8..000000000 --- a/.github/workflows/security_validation.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Security validation - -on: - pull_request: - branches: - - 'develop' - - 'main' - -jobs: - security: - name: Check for vulnerabilities - runs-on: ubuntu-latest - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - steps: - - uses: actions/checkout@v2 - - - uses: snyk/actions/setup@master - - name: Set up Go 1.16 - uses: actions/setup-go@v2 - with: - go-version: '1.16' - - - name: Run snyl on all projects - run: snyk test --all-projects diff --git a/.gitignore b/.gitignore index 4d5c55b7e..18b659878 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,6 @@ # vendor/ .idea/ build -*.db # Mac OS .DS_Store @@ -29,3 +28,10 @@ build # pprof pprof/* + +# Database Files +*.bin +*.gob + +# Nohup Files - https://man7.org/linux/man-pages/man1/nohup.1p.html +nohup.* diff --git a/Dockerfile b/Dockerfile index 65b7b46ea..51a42eae9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ FROM golang:1.16-alpine AS builder # Set necessary environment variables needed for our image. ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64 -RUN apk add libpcap-dev gcc g++ make bash +RUN apk add libpcap-dev gcc g++ make bash perl-utils # Move to agent working directory (/agent-build). WORKDIR /app/agent-build @@ -24,7 +24,7 @@ COPY tap/go.mod tap/go.mod ../tap/ COPY tap/api/go.* ../tap/api/ RUN go mod download # cheap trick to make the build faster (As long as go.mod wasn't changes) -RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get +RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get ARG COMMIT_HASH ARG GIT_BRANCH @@ -41,16 +41,24 @@ RUN go build -ldflags="-s -w \ -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \ -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent . +# Download Basenine executable, verify the sha1sum and move it to a directory in $PATH +ADD https://github.com/up9inc/basenine/releases/download/v0.2.19/basenine_linux_amd64 ./basenine_linux_amd64 +ADD https://github.com/up9inc/basenine/releases/download/v0.2.19/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256 +RUN shasum -a 256 -c basenine_linux_amd64.sha256 +RUN chmod +x ./basenine_linux_amd64 + COPY devops/build_extensions.sh .. RUN cd .. && /bin/bash build_extensions.sh FROM alpine:3.14 -RUN apk add bash libpcap-dev tcpdump +RUN apk add bash libpcap-dev + WORKDIR /app # Copy binary and config files from /build to root folder of scratch container. COPY --from=builder ["/app/agent-build/mizuagent", "."] +COPY --from=builder ["/app/agent-build/basenine_linux_amd64", "/usr/local/bin/basenine"] COPY --from=builder ["/app/agent/build/extensions", "extensions"] COPY --from=site-build ["/app/ui-build/build", "site"] RUN mkdir /app/data/ diff --git a/README.md b/README.md index 3c31c489e..56d537899 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,21 @@ A simple-yet-powerful API traffic viewer for Kubernetes enabling you to view all API communication between microservices to help your debug and troubleshoot regressions. -Think TCPDump and Chrome Dev Tools combined. +Think TCPDump and Wireshark re-invented for Kubernetes. ![Simple UI](assets/mizu-ui.png) ## Features - Simple and powerful CLI -- Real-time view of all HTTP requests, REST and gRPC API calls -- No installation or code instrumentation -- Works completely on premises +- Monitoring network traffic in real-time. Supported protocols: + - [HTTP/1.1](https://datatracker.ietf.org/doc/html/rfc2616) (REST, etc.) + - [HTTP/2](https://datatracker.ietf.org/doc/html/rfc7540) (gRPC) + - [AMQP](https://www.rabbitmq.com/amqp-0-9-1-reference.html) (RabbitMQ, Apache Qpid, etc.) + - [Apache Kafka](https://kafka.apache.org/protocol) + - [Redis](https://redis.io/topics/protocol) +- Works with Kubernetes APIs. No installation or code instrumentation +- Rich filtering ## Requirements @@ -44,15 +49,6 @@ SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/ ### Development (unstable) Build Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page -## Kubeconfig & Permissions -While `mizu`most often works out of the box, you can influence its behavior: - -1. [OPTIONAL] Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config` -2. `mizu` assumes user running the command has permissions to create resources (such as pods, services, namespaces) on your Kubernetes cluster (no worries - `mizu` resources are cleaned up upon termination) - -For detailed list of k8s permissions see [PERMISSIONS](docs/PERMISSIONS.md) document - - ## How to Run 1. Find pods you'd like to tap to in your Kubernetes cluster @@ -83,7 +79,7 @@ To tap all pods in current namespace - ``` -To tap specific pod - +### To tap specific pod ```bash $ kubectl get pods NAME READY STATUS RESTARTS AGE @@ -96,7 +92,7 @@ To tap specific pod - ^C ``` -To tap multiple pods using regex - +### To tap multiple pods using regex ```bash $ kubectl get pods NAME READY STATUS RESTARTS AGE @@ -114,21 +110,22 @@ To tap multiple pods using regex - ## Configuration -Mizu can work with config file which should be stored in ${HOME}/.mizu/config.yaml (macOS: ~/.mizu/config.yaml)
-In case no config file found, defaults will be used
+Mizu can optionally work with a config file that can be provided as a CLI argument (using `--set config-path=`) or if not provided, will be stored at ${HOME}/.mizu/config.yaml In case of partial configuration defined, all other fields will be used with defaults
You can always override the defaults or config file with CLI flags To get the default config params run `mizu config`
To generate a new config file with default values use `mizu config -r` -### Telemetry - -By default, mizu reports usage telemetry. It can be disabled by adding a line of `telemetry: false` in the `${HOME}/.mizu/config.yaml` file - ## Advanced Usage +### Kubeconfig + +It is possible to change the kubeconfig path using `KUBECONFIG` environment variable or the command like flag +with `--set kube-config-path=`.
+If both are not set - Mizu assumes that configuration is at `${HOME}/.kube/config` + ### Namespace-Restricted Mode Some users have permission to only manage resources in one particular namespace assigned to them @@ -142,6 +139,8 @@ using the `--namespace` flag or by setting `tap.namespaces` in the config file Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior +For detailed list of k8s permissions see [PERMISSIONS](docs/PERMISSIONS.md) document + ### User agent filtering User-agent filtering (like health checks) - can be configured using command-line options: @@ -182,23 +181,7 @@ and when changed it will support accessing by IP ### Run in daemon mode -Mizu can be ran detached from the cli using the daemon flag: `mizu tap --daemon`. This type of mizu instance will run indefinitely in the cluster. +Mizu can be run detached from the cli using the daemon flag: `mizu tap --daemon`. This type of mizu instance will run +indefinitely in the cluster. -Please note that daemon mode requires you to have RBAC creation permissions, see the [permissions](docs/PERMISSIONS.md) doc for more details. - -In order to access a daemon mizu you will have to run `mizu view` after running the `tap --daemon` command. - -To stop the detached mizu instance and clean all cluster side resources, run `mizu clean` - - -## How to Run local UI - -- run from mizu/agent `go run main.go --hars-read --hars-dir ` - -- copy Har files into the folder from last command - -- change `MizuWebsocketURL` and `apiURL` in `api.js` file - -- run from mizu/ui - `npm run start` - -- open browser on `localhost:3000` +For more information please refer to [DAEMON MODE](docs/DAEMON_MODE.md) diff --git a/acceptanceTests/Makefile b/acceptanceTests/Makefile index c6d411544..9e7c7f0e0 100644 --- a/acceptanceTests/Makefile +++ b/acceptanceTests/Makefile @@ -1,2 +1,2 @@ test: ## Run acceptance tests. - @go test ./... -timeout 1h + @go test ./... -timeout 1h -v diff --git a/acceptanceTests/config_test.go b/acceptanceTests/config_test.go index 248e56e9e..b0abed868 100644 --- a/acceptanceTests/config_test.go +++ b/acceptanceTests/config_test.go @@ -2,11 +2,12 @@ package acceptanceTests import ( "fmt" - "gopkg.in/yaml.v3" "io/ioutil" "os" "os/exec" "testing" + + "gopkg.in/yaml.v3" ) type tapConfig struct { diff --git a/acceptanceTests/go.mod b/acceptanceTests/go.mod index 0ced4f361..3bba434f2 100644 --- a/acceptanceTests/go.mod +++ b/acceptanceTests/go.mod @@ -3,6 +3,7 @@ module github.com/up9inc/mizu/tests go 1.16 require ( + github.com/gorilla/websocket v1.4.2 github.com/up9inc/mizu/shared v0.0.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/acceptanceTests/go.sum b/acceptanceTests/go.sum index 54a1e835d..808c5bc66 100644 --- a/acceptanceTests/go.sum +++ b/acceptanceTests/go.sum @@ -211,6 +211,7 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -303,6 +304,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= diff --git a/acceptanceTests/tap_test.go b/acceptanceTests/tap_test.go index 7bd7805d7..51b57a4ea 100644 --- a/acceptanceTests/tap_test.go +++ b/acceptanceTests/tap_test.go @@ -66,21 +66,18 @@ func TestTap(t *testing.T) { entriesCheckFunc := func() error { timestamp := time.Now().UnixNano() / int64(time.Millisecond) - entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt×tamp=%v", apiServerUrl, entriesCount, timestamp) - requestResult, requestErr := executeHttpGetRequest(entriesUrl) - if requestErr != nil { - return fmt.Errorf("failed to get entries, err: %v", requestErr) + entries, err := getDBEntries(timestamp, entriesCount, 1*time.Second) + if err != nil { + return err } - - entries := requestResult.([]interface{}) - if len(entries) == 0 { - return fmt.Errorf("unexpected entries result - Expected more than 0 entries") + err = checkEntriesAtLeast(entries, 1) + if err != nil { + return err } - - entry := entries[0].(map[string]interface{}) + entry := entries[0] entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entry["id"]) - requestResult, requestErr = executeHttpGetRequest(entryUrl) + requestResult, requestErr := executeHttpGetRequest(entryUrl) if requestErr != nil { return fmt.Errorf("failed to get entry, err: %v", requestErr) } @@ -150,10 +147,7 @@ func TestTapAllNamespaces(t *testing.T) { t.Skip("ignored acceptance test") } - expectedPods := []struct{ - Name string - Namespace string - }{ + expectedPods := []PodDescriptor{ {Name: "httpbin", Namespace: "mizu-tests"}, {Name: "httpbin", Namespace: "mizu-tests2"}, } @@ -202,19 +196,7 @@ func TestTapAllNamespaces(t *testing.T) { } for _, expectedPod := range expectedPods { - podFound := false - - for _, pod := range pods { - podNamespace := pod["namespace"].(string) - podName := pod["name"].(string) - - if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) { - podFound = true - break - } - } - - if !podFound { + if !isPodDescriptorInPodArray(pods, expectedPod) { t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name) return } @@ -226,10 +208,7 @@ func TestTapMultipleNamespaces(t *testing.T) { t.Skip("ignored acceptance test") } - expectedPods := []struct{ - Name string - Namespace string - }{ + expectedPods := []PodDescriptor{ {Name: "httpbin", Namespace: "mizu-tests"}, {Name: "httpbin2", Namespace: "mizu-tests"}, {Name: "httpbin", Namespace: "mizu-tests2"}, @@ -288,19 +267,7 @@ func TestTapMultipleNamespaces(t *testing.T) { } for _, expectedPod := range expectedPods { - podFound := false - - for _, pod := range pods { - podNamespace := pod["namespace"].(string) - podName := pod["name"].(string) - - if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) { - podFound = true - break - } - } - - if !podFound { + if !isPodDescriptorInPodArray(pods, expectedPod) { t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name) return } @@ -313,10 +280,7 @@ func TestTapRegex(t *testing.T) { } regexPodName := "httpbin2" - expectedPods := []struct{ - Name string - Namespace string - }{ + expectedPods := []PodDescriptor{ {Name: regexPodName, Namespace: "mizu-tests"}, } @@ -371,19 +335,7 @@ func TestTapRegex(t *testing.T) { } for _, expectedPod := range expectedPods { - podFound := false - - for _, pod := range pods { - podNamespace := pod["namespace"].(string) - podName := pod["name"].(string) - - if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) { - podFound = true - break - } - } - - if !podFound { + if !isPodDescriptorInPodArray(pods, expectedPod) { t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name) return } @@ -431,7 +383,7 @@ func TestTapDryRun(t *testing.T) { resultChannel <- "fail" }() - testResult := <- resultChannel + testResult := <-resultChannel if testResult != "success" { t.Errorf("unexpected result - dry run cmd not done") } @@ -475,9 +427,10 @@ func TestTapRedact(t *testing.T) { } proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName) + requestHeaders := map[string]string{"User-Header": "Mizu"} requestBody := map[string]string{"User": "Mizu"} for i := 0; i < defaultEntriesCount; i++ { - if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil { + if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil { t.Errorf("failed to send proxy request, err: %v", requestErr) return } @@ -486,51 +439,39 @@ func TestTapRedact(t *testing.T) { redactCheckFunc := func() error { timestamp := time.Now().UnixNano() / int64(time.Millisecond) - entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt×tamp=%v", apiServerUrl, defaultEntriesCount, timestamp) - requestResult, requestErr := executeHttpGetRequest(entriesUrl) - if requestErr != nil { - return fmt.Errorf("failed to get entries, err: %v", requestErr) + entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second) + if err != nil { + return err } - - entries := requestResult.([]interface{}) - if len(entries) == 0 { - return fmt.Errorf("unexpected entries result - Expected more than 0 entries") + err = checkEntriesAtLeast(entries, 1) + if err != nil { + return err } - - firstEntry := entries[0].(map[string]interface{}) + firstEntry := entries[0] entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"]) - requestResult, requestErr = executeHttpGetRequest(entryUrl) + requestResult, requestErr := executeHttpGetRequest(entryUrl) if requestErr != nil { return fmt.Errorf("failed to get entry, err: %v", requestErr) } - data := requestResult.(map[string]interface{})["data"].(map[string]interface{}) - entryJson := data["entry"].(string) + entry := requestResult.(map[string]interface{})["data"].(map[string]interface{}) + request := entry["request"].(map[string]interface{}) - var entry map[string]interface{} - if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil { - return fmt.Errorf("failed to parse entry, err: %v", parseErr) - } - - entryRequest := entry["request"].(map[string]interface{}) - entryPayload := entryRequest["payload"].(map[string]interface{}) - entryDetails := entryPayload["details"].(map[string]interface{}) - - headers := entryDetails["headers"].([]interface{}) + headers := request["_headers"].([]interface{}) for _, headerInterface := range headers { header := headerInterface.(map[string]interface{}) - if header["name"].(string) != "User-Agent" { + if header["name"].(string) != "User-Header" { continue } - userAgent := header["value"].(string) - if userAgent != "[REDACTED]" { + userHeader := header["value"].(string) + if userHeader != "[REDACTED]" { return fmt.Errorf("unexpected result - user agent is not redacted") } } - postData := entryDetails["postData"].(map[string]interface{}) + postData := request["postData"].(map[string]interface{}) textDataStr := postData["text"].(string) var textData map[string]string @@ -590,9 +531,10 @@ func TestTapNoRedact(t *testing.T) { } proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName) + requestHeaders := map[string]string{"User-Header": "Mizu"} requestBody := map[string]string{"User": "Mizu"} for i := 0; i < defaultEntriesCount; i++ { - if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil { + if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil { t.Errorf("failed to send proxy request, err: %v", requestErr) return } @@ -601,51 +543,39 @@ func TestTapNoRedact(t *testing.T) { redactCheckFunc := func() error { timestamp := time.Now().UnixNano() / int64(time.Millisecond) - entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt×tamp=%v", apiServerUrl, defaultEntriesCount, timestamp) - requestResult, requestErr := executeHttpGetRequest(entriesUrl) - if requestErr != nil { - return fmt.Errorf("failed to get entries, err: %v", requestErr) + entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second) + if err != nil { + return err } - - entries := requestResult.([]interface{}) - if len(entries) == 0 { - return fmt.Errorf("unexpected entries result - Expected more than 0 entries") + err = checkEntriesAtLeast(entries, 1) + if err != nil { + return err } - - firstEntry := entries[0].(map[string]interface{}) + firstEntry := entries[0] entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"]) - requestResult, requestErr = executeHttpGetRequest(entryUrl) + requestResult, requestErr := executeHttpGetRequest(entryUrl) if requestErr != nil { return fmt.Errorf("failed to get entry, err: %v", requestErr) } - data := requestResult.(map[string]interface{})["data"].(map[string]interface{}) - entryJson := data["entry"].(string) + entry := requestResult.(map[string]interface{})["data"].(map[string]interface{}) + request := entry["request"].(map[string]interface{}) - var entry map[string]interface{} - if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil { - return fmt.Errorf("failed to parse entry, err: %v", parseErr) - } - - entryRequest := entry["request"].(map[string]interface{}) - entryPayload := entryRequest["payload"].(map[string]interface{}) - entryDetails := entryPayload["details"].(map[string]interface{}) - - headers := entryDetails["headers"].([]interface{}) + headers := request["_headers"].([]interface{}) for _, headerInterface := range headers { header := headerInterface.(map[string]interface{}) - if header["name"].(string) != "User-Agent" { + if header["name"].(string) != "User-Header" { continue } - userAgent := header["value"].(string) - if userAgent == "[REDACTED]" { + userHeader := header["value"].(string) + if userHeader == "[REDACTED]" { return fmt.Errorf("unexpected result - user agent is redacted") } } - postData := entryDetails["postData"].(map[string]interface{}) + postData := request["postData"].(map[string]interface{}) textDataStr := postData["text"].(string) var textData map[string]string @@ -716,38 +646,26 @@ func TestTapRegexMasking(t *testing.T) { redactCheckFunc := func() error { timestamp := time.Now().UnixNano() / int64(time.Millisecond) - entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt×tamp=%v", apiServerUrl, defaultEntriesCount, timestamp) - requestResult, requestErr := executeHttpGetRequest(entriesUrl) - if requestErr != nil { - return fmt.Errorf("failed to get entries, err: %v", requestErr) + entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second) + if err != nil { + return err } - - entries := requestResult.([]interface{}) - if len(entries) == 0 { - return fmt.Errorf("unexpected entries result - Expected more than 0 entries") + err = checkEntriesAtLeast(entries, 1) + if err != nil { + return err } - - firstEntry := entries[0].(map[string]interface{}) + firstEntry := entries[0] entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"]) - requestResult, requestErr = executeHttpGetRequest(entryUrl) + requestResult, requestErr := executeHttpGetRequest(entryUrl) if requestErr != nil { return fmt.Errorf("failed to get entry, err: %v", requestErr) } - data := requestResult.(map[string]interface{})["data"].(map[string]interface{}) - entryJson := data["entry"].(string) + entry := requestResult.(map[string]interface{})["data"].(map[string]interface{}) + request := entry["request"].(map[string]interface{}) - var entry map[string]interface{} - if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil { - return fmt.Errorf("failed to parse entry, err: %v", parseErr) - } - - entryRequest := entry["request"].(map[string]interface{}) - entryPayload := entryRequest["payload"].(map[string]interface{}) - entryDetails := entryPayload["details"].(map[string]interface{}) - - postData := entryDetails["postData"].(map[string]interface{}) + postData := request["postData"].(map[string]interface{}) textData := postData["text"].(string) if textData != "[REDACTED]" { @@ -805,7 +723,7 @@ func TestTapIgnoredUserAgents(t *testing.T) { proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName) ignoredUserAgentCustomHeader := "Ignored-User-Agent" - headers := map[string]string {"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""} + headers := map[string]string{"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""} for i := 0; i < defaultEntriesCount; i++ { if _, requestErr := executeHttpGetRequestWithHeaders(fmt.Sprintf("%v/get", proxyUrl), headers); requestErr != nil { t.Errorf("failed to send proxy request, err: %v", requestErr) @@ -823,38 +741,27 @@ func TestTapIgnoredUserAgents(t *testing.T) { ignoredUserAgentsCheckFunc := func() error { timestamp := time.Now().UnixNano() / int64(time.Millisecond) - entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt×tamp=%v", apiServerUrl, defaultEntriesCount * 2, timestamp) - requestResult, requestErr := executeHttpGetRequest(entriesUrl) - if requestErr != nil { - return fmt.Errorf("failed to get entries, err: %v", requestErr) + entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second) + if err != nil { + return err } - - entries := requestResult.([]interface{}) - if len(entries) == 0 { - return fmt.Errorf("unexpected entries result - Expected more than 0 entries") + err = checkEntriesAtLeast(entries, 1) + if err != nil { + return err } for _, entryInterface := range entries { - entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entryInterface.(map[string]interface{})["id"]) - requestResult, requestErr = executeHttpGetRequest(entryUrl) + entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entryInterface["id"]) + requestResult, requestErr := executeHttpGetRequest(entryUrl) if requestErr != nil { return fmt.Errorf("failed to get entry, err: %v", requestErr) } - data := requestResult.(map[string]interface{})["data"].(map[string]interface{}) - entryJson := data["entry"].(string) + entry := requestResult.(map[string]interface{})["data"].(map[string]interface{}) + request := entry["request"].(map[string]interface{}) - var entry map[string]interface{} - if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil { - return fmt.Errorf("failed to parse entry, err: %v", parseErr) - } - - entryRequest := entry["request"].(map[string]interface{}) - entryPayload := entryRequest["payload"].(map[string]interface{}) - entryDetails := entryPayload["details"].(map[string]interface{}) - - entryHeaders := entryDetails["headers"].([]interface{}) - for _, headerInterface := range entryHeaders { + headers := request["_headers"].([]interface{}) + for _, headerInterface := range headers { header := headerInterface.(map[string]interface{}) if header["name"].(string) != ignoredUserAgentCustomHeader { continue @@ -922,21 +829,21 @@ func TestTapDumpLogs(t *testing.T) { return } - var dumpsLogsPath string + var dumpLogsPath string for _, file := range files { fileName := file.Name() if strings.Contains(fileName, "mizu_logs") { - dumpsLogsPath = path.Join(mizuFolderPath, fileName) + dumpLogsPath = path.Join(mizuFolderPath, fileName) break } } - if dumpsLogsPath == "" { + if dumpLogsPath == "" { t.Errorf("dump logs file not found") return } - zipReader, zipError := zip.OpenReader(dumpsLogsPath) + zipReader, zipError := zip.OpenReader(dumpLogsPath) if zipError != nil { t.Errorf("failed to get zip reader, err: %v", zipError) return @@ -973,3 +880,251 @@ func TestTapDumpLogs(t *testing.T) { return } } + +func TestDaemonSeeTraffic(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + tests := []int{50} + + for _, entriesCount := range tests { + t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) { + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + tapDaemonCmdArgs := getDefaultTapCommandArgsWithDaemonMode() + + tapNamespace := getDefaultTapNamespace() + tapDaemonCmdArgs = append(tapDaemonCmdArgs, tapNamespace...) + + tapCmd := exec.Command(cliPath, tapDaemonCmdArgs...) + + viewCmd := exec.Command(cliPath, getDefaultViewCommandArgs()...) + + t.Cleanup(func() { + daemonCleanup(t, viewCmd) + }) + + t.Logf("running command: %v", tapCmd.String()) + if err := tapCmd.Run(); err != nil { + t.Errorf("error occured while running the tap command, err: %v", err) + return + } + + t.Logf("running command: %v", viewCmd.String()) + if err := viewCmd.Start(); err != nil { + t.Errorf("error occured while running the view command, err: %v", err) + return + } + + apiServerUrl := getApiServerUrl(defaultApiServerPort) + if err := waitTapPodsReady(apiServerUrl); err != nil { + t.Errorf("failed to start tap pods on time, err: %v", err) + return + } + + proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName) + for i := 0; i < entriesCount; i++ { + if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil { + t.Errorf("failed to send proxy request, err: %v", requestErr) + return + } + } + + entriesCheckFunc := func() error { + timestamp := time.Now().UnixNano() / int64(time.Millisecond) + + entries, err := getDBEntries(timestamp, entriesCount, 1*time.Second) + if err != nil { + return err + } + err = checkEntriesAtLeast(entries, 1) + if err != nil { + return err + } + entry := entries[0] + + entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entry["id"]) + requestResult, requestErr := executeHttpGetRequest(entryUrl) + if requestErr != nil { + return fmt.Errorf("failed to get entry, err: %v", requestErr) + } + + if requestResult == nil { + return fmt.Errorf("unexpected nil entry result") + } + + return nil + } + if err := retriesExecute(shortRetriesCount, entriesCheckFunc); err != nil { + t.Errorf("%v", err) + return + } + }) + } +} + +func TestDaemonMultipleNamespacesSeePods(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + expectedPods := []PodDescriptor{ + {Name: "httpbin", Namespace: "mizu-tests"}, + {Name: "httpbin2", Namespace: "mizu-tests"}, + {Name: "httpbin", Namespace: "mizu-tests2"}, + } + + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + tapCmdArgs := getDefaultTapCommandArgsWithDaemonMode() + var namespacesCmd []string + for _, expectedPod := range expectedPods { + namespacesCmd = append(namespacesCmd, "-n", expectedPod.Namespace) + } + tapCmdArgs = append(tapCmdArgs, namespacesCmd...) + + tapCmd := exec.Command(cliPath, tapCmdArgs...) + + viewCmd := exec.Command(cliPath, getDefaultViewCommandArgs()...) + + t.Cleanup(func() { + daemonCleanup(t, viewCmd) + }) + + t.Logf("running command: %v", tapCmd.String()) + if err := tapCmd.Run(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + t.Logf("running command: %v", viewCmd.String()) + if err := viewCmd.Start(); err != nil { + t.Errorf("error occured while running the view command, err: %v", err) + return + } + + apiServerUrl := getApiServerUrl(defaultApiServerPort) + if err := waitTapPodsReady(apiServerUrl); err != nil { + t.Errorf("failed to start tap pods on time, err: %v", err) + return + } + + podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl) + requestResult, requestErr := executeHttpGetRequest(podsUrl) + if requestErr != nil { + t.Errorf("failed to get tap status, err: %v", requestErr) + return + } + + pods, err := getPods(requestResult) + if err != nil { + t.Errorf("failed to get pods, err: %v", err) + return + } + + if len(expectedPods) != len(pods) { + t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods)) + return + } + + for _, expectedPod := range expectedPods { + if !isPodDescriptorInPodArray(pods, expectedPod) { + t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name) + return + } + } +} + +func TestDaemonSingleNamespaceSeePods(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + expectedPods := []PodDescriptor{ + {Name: "httpbin", Namespace: "mizu-tests"}, + {Name: "httpbin2", Namespace: "mizu-tests"}, + } + unexpectedPods := []PodDescriptor{ + {Name: "httpbin", Namespace: "mizu-tests2"}, + } + + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + tapCmdArgs := getDefaultTapCommandArgsWithDaemonMode() + var namespacesCmd []string + for _, expectedPod := range expectedPods { + namespacesCmd = append(namespacesCmd, "-n", expectedPod.Namespace) + } + tapCmdArgs = append(tapCmdArgs, namespacesCmd...) + + tapCmd := exec.Command(cliPath, tapCmdArgs...) + + viewCmd := exec.Command(cliPath, getDefaultViewCommandArgs()...) + + t.Cleanup(func() { + daemonCleanup(t, viewCmd) + }) + + t.Logf("running command: %v", tapCmd.String()) + if err := tapCmd.Run(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + t.Logf("running command: %v", viewCmd.String()) + if err := viewCmd.Start(); err != nil { + t.Errorf("error occured while running the view command, err: %v", err) + return + } + + apiServerUrl := getApiServerUrl(defaultApiServerPort) + if err := waitTapPodsReady(apiServerUrl); err != nil { + t.Errorf("failed to start tap pods on time, err: %v", err) + return + } + + podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl) + requestResult, requestErr := executeHttpGetRequest(podsUrl) + if requestErr != nil { + t.Errorf("failed to get tap status, err: %v", requestErr) + return + } + + pods, err := getPods(requestResult) + if err != nil { + t.Errorf("failed to get pods, err: %v", err) + return + } + + for _, unexpectedPod := range unexpectedPods { + if isPodDescriptorInPodArray(pods, unexpectedPod) { + t.Errorf("unexpected result - unexpected pod found, pod namespace: %v, pod name: %v", unexpectedPod.Namespace, unexpectedPod.Name) + return + } + } + + if len(expectedPods) != len(pods) { + t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods)) + return + } + + for _, expectedPod := range expectedPods { + if !isPodDescriptorInPodArray(pods, expectedPod) { + t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name) + return + } + } +} diff --git a/acceptanceTests/testsUtils.go b/acceptanceTests/testsUtils.go index 9c7ad955a..715df51d7 100644 --- a/acceptanceTests/testsUtils.go +++ b/acceptanceTests/testsUtils.go @@ -3,6 +3,7 @@ package acceptanceTests import ( "bytes" "encoding/json" + "errors" "fmt" "io/ioutil" "net/http" @@ -10,9 +11,12 @@ import ( "os/exec" "path" "strings" + "sync" "syscall" + "testing" "time" + "github.com/gorilla/websocket" "github.com/up9inc/mizu/shared" ) @@ -24,8 +28,26 @@ const ( defaultServiceName = "httpbin" defaultEntriesCount = 50 waitAfterTapPodsReady = 3 * time.Second + cleanCommandTimeout = 1 * time.Minute ) +type PodDescriptor struct { + Name string + Namespace string +} + +func isPodDescriptorInPodArray(pods []map[string]interface{}, podDescriptor PodDescriptor) bool { + for _, pod := range pods { + podNamespace := pod["namespace"].(string) + podName := pod["name"].(string) + + if podDescriptor.Namespace == podNamespace && strings.Contains(podName, podDescriptor.Name) { + return true + } + } + return false +} + func getCliPath() (string, error) { dir, filePathErr := os.Getwd() if filePathErr != nil { @@ -59,7 +81,11 @@ func getProxyUrl(namespace string, service string) string { } func getApiServerUrl(port uint16) string { - return fmt.Sprintf("http://localhost:%v/mizu", port) + return fmt.Sprintf("http://localhost:%v", port) +} + +func getWebSocketUrl(port uint16) string { + return fmt.Sprintf("ws://localhost:%v/ws", port) } func getDefaultCommandArgs() []string { @@ -67,8 +93,9 @@ func getDefaultCommandArgs() []string { telemetry := "telemetry=false" agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0" imagePullPolicy := "image-pull-policy=Never" + headless := "headless=true" - return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy} + return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless} } func getDefaultTapCommandArgs() []string { @@ -78,6 +105,10 @@ func getDefaultTapCommandArgs() []string { return append([]string{tapCommand}, defaultCmdArgs...) } +func getDefaultTapCommandArgsWithDaemonMode() []string { + return append(getDefaultTapCommandArgs(), "--daemon") +} + func getDefaultTapCommandArgsWithRegex(regex string) []string { tapCommand := "tap" defaultCmdArgs := getDefaultCommandArgs() @@ -103,6 +134,20 @@ func getDefaultConfigCommandArgs() []string { return append([]string{configCommand}, defaultCmdArgs...) } +func getDefaultCleanCommandArgs() []string { + cleanCommand := "clean" + defaultCmdArgs := getDefaultCommandArgs() + + return append([]string{cleanCommand}, defaultCmdArgs...) +} + +func getDefaultViewCommandArgs() []string { + viewCommand := "view" + defaultCmdArgs := getDefaultCommandArgs() + + return append([]string{viewCommand}, defaultCmdArgs...) +} + func retriesExecute(retriesCount int, executeFunc func() error) error { var lastError interface{} @@ -195,16 +240,57 @@ func executeHttpGetRequest(url string) (interface{}, error) { return executeHttpRequest(response, requestErr) } -func executeHttpPostRequest(url string, body interface{}) (interface{}, error) { +func executeHttpPostRequestWithHeaders(url string, headers map[string]string, body interface{}) (interface{}, error) { requestBody, jsonErr := json.Marshal(body) if jsonErr != nil { return nil, jsonErr } - response, requestErr := http.Post(url, "application/json", bytes.NewBuffer(requestBody)) + request, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(requestBody)) + if err != nil { + return nil, err + } + + request.Header.Add("Content-Type", "application/json") + for headerKey, headerValue := range headers { + request.Header.Add(headerKey, headerValue) + } + + client := &http.Client{} + response, requestErr := client.Do(request) return executeHttpRequest(response, requestErr) } +func runMizuClean() error { + cliPath, err := getCliPath() + if err != nil { + return err + } + + cleanCmdArgs := getDefaultCleanCommandArgs() + + cleanCmd := exec.Command(cliPath, cleanCmdArgs...) + + commandDone := make(chan error) + go func() { + if err := cleanCmd.Run(); err != nil { + commandDone <- err + } + commandDone <- nil + }() + + select { + case err = <-commandDone: + if err != nil { + return err + } + case <-time.After(cleanCommandTimeout): + return errors.New("clean command timed out") + } + + return nil +} + func cleanupCommand(cmd *exec.Cmd) error { if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil { return err @@ -218,11 +304,10 @@ func cleanupCommand(cmd *exec.Cmd) error { } func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) { - tapStatus := tapStatusInterface.(map[string]interface{}) - podsInterface := tapStatus["pods"].([]interface{}) + tapPodsInterface := tapStatusInterface.([]interface{}) var pods []map[string]interface{} - for _, podInterface := range podsInterface { + for _, podInterface := range tapPodsInterface { pods = append(pods, podInterface.(map[string]interface{})) } @@ -239,6 +324,87 @@ func getLogsPath() (string, error) { return logsPath, nil } +func daemonCleanup(t *testing.T, viewCmd *exec.Cmd) { + if err := runMizuClean(); err != nil { + t.Logf("error running mizu clean: %v", err) + } + + if err := cleanupCommand(viewCmd); err != nil { + t.Logf("failed to cleanup view command, err: %v", err) + } +} + +// waitTimeout waits for the waitgroup for the specified max timeout. +// Returns true if waiting timed out. +func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool { + channel := make(chan struct{}) + go func() { + defer close(channel) + wg.Wait() + }() + select { + case <-channel: + return false // completed normally + case <-time.After(timeout): + return true // timed out + } +} + +// checkEntriesAtLeast checks whether the number of entries greater than or equal to n +func checkEntriesAtLeast(entries []map[string]interface{}, n int) error { + if len(entries) < n { + return fmt.Errorf("Unexpected entries result - Expected more than %d entries", n-1) + } + return nil +} + +// getDBEntries retrieves the entries from the database before the given timestamp. +// Also limits the results according to the limit parameter. +// Timeout for the WebSocket connection is defined by the timeout parameter. +func getDBEntries(timestamp int64, limit int, timeout time.Duration) (entries []map[string]interface{}, err error) { + query := fmt.Sprintf("timestamp < %d and limit(%d)", timestamp, limit) + webSocketUrl := getWebSocketUrl(defaultApiServerPort) + + var connection *websocket.Conn + connection, _, err = websocket.DefaultDialer.Dial(webSocketUrl, nil) + if err != nil { + return + } + defer connection.Close() + + handleWSConnection := func(wg *sync.WaitGroup) { + defer wg.Done() + for { + _, message, err := connection.ReadMessage() + if err != nil { + return + } + + var data map[string]interface{} + if err = json.Unmarshal([]byte(message), &data); err != nil { + return + } + + if data["messageType"] == "entry" { + entries = append(entries, data) + } + } + } + + err = connection.WriteMessage(websocket.TextMessage, []byte(query)) + if err != nil { + return + } + + var wg sync.WaitGroup + go handleWSConnection(&wg) + wg.Add(1) + + waitTimeout(&wg, timeout) + + return +} + func Contains(slice []string, containsValue string) bool { for _, sliceValue := range slice { if sliceValue == containsValue { diff --git a/agent/go.mod b/agent/go.mod index 905c80f0b..a4ccbb693 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -3,11 +3,11 @@ module mizuserver go 1.16 require ( + github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b github.com/djherbis/atime v1.0.0 - github.com/fsnotify/fsnotify v1.4.9 github.com/getkin/kin-openapi v0.76.0 github.com/gin-contrib/static v0.0.1 - github.com/gin-gonic/gin v1.7.2 + github.com/gin-gonic/gin v1.7.7 github.com/go-playground/locales v0.13.0 github.com/go-playground/universal-translator v0.17.0 github.com/go-playground/validator/v10 v10.5.0 @@ -16,13 +16,12 @@ require ( github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 github.com/patrickmn/go-cache v2.1.0+incompatible + github.com/up9inc/basenine/client/go v0.0.0-20211215185650-10083bb9a1b3 github.com/up9inc/mizu/shared v0.0.0 github.com/up9inc/mizu/tap v0.0.0 github.com/up9inc/mizu/tap/api v0.0.0 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 - go.mongodb.org/mongo-driver v1.7.1 - gorm.io/driver/sqlite v1.1.4 - gorm.io/gorm v1.21.8 + golang.org/x/text v0.3.5 // indirect k8s.io/api v0.21.2 k8s.io/apimachinery v0.21.2 k8s.io/client-go v0.21.2 diff --git a/agent/go.sum b/agent/go.sum index 3994ea223..ed288cfc1 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -52,6 +52,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b h1:8m+eVxVVDDyJFidv7Ck1OwqnDaQR6pTSRGlCC2Dnw0A= +github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b/go.mod h1:+tQQjzrp2501Nd6JXrb9s/XsNvFK3ZbxOnCdQl/vDRo= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -94,7 +96,6 @@ github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg= github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= @@ -111,7 +112,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/getkin/kin-openapi v0.76.0 h1:j77zg3Ec+k+r+GA3d8hBoXpAc6KX9TbBPrwQGBIy2sY= @@ -125,6 +125,8 @@ github.com/gin-contrib/static v0.0.1/go.mod h1:CSxeF+wep05e0kCOsqWdAWbSszmc31zTI github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gin-gonic/gin v1.7.2 h1:Tg03T9yM2xa8j6I3Z3oqLaQRSmKvxPd6g/2HJ6zICFA= github.com/gin-gonic/gin v1.7.2/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= +github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= +github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= @@ -195,31 +197,7 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E= github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -251,7 +229,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -321,12 +298,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= -github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= -github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -336,14 +307,10 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -364,16 +331,12 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ= -github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -395,7 +358,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= @@ -419,7 +381,6 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -447,8 +408,6 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -458,8 +417,6 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -468,7 +425,6 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -485,8 +441,9 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -495,24 +452,22 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/up9inc/basenine/client/go v0.0.0-20211215185650-10083bb9a1b3 h1:FeDCVOBFVpZA5/O5hfPdGTn0rdR2jTEYo3iB2htELI4= +github.com/up9inc/basenine/client/go v0.0.0-20211215185650-10083bb9a1b3/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.7.1 h1:jwqTeEM3x6L9xDXrCxN0Hbg7vdGfPBOTIkr0+/LYZDA= -go.mongodb.org/mongo-driver v1.7.1/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -525,13 +480,11 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= @@ -609,7 +562,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -623,13 +575,10 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -646,6 +595,7 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -654,7 +604,6 @@ golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -683,13 +632,9 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -803,11 +748,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM= -gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw= -gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.21.8 h1:2CEwZSzogdhsKPlJ9OvBKTdlWIpELXb6HbfLfMNhSYI= -gorm.io/gorm v1.21.8/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/agent/main.go b/agent/main.go index 51ffc9a41..f90dc0f96 100644 --- a/agent/main.go +++ b/agent/main.go @@ -6,13 +6,10 @@ import ( "errors" "flag" "fmt" - "github.com/up9inc/mizu/shared/kubernetes" "io/ioutil" - v1 "k8s.io/api/core/v1" "mizuserver/pkg/api" "mizuserver/pkg/config" "mizuserver/pkg/controllers" - "mizuserver/pkg/database" "mizuserver/pkg/models" "mizuserver/pkg/providers" "mizuserver/pkg/routes" @@ -20,6 +17,7 @@ import ( "mizuserver/pkg/utils" "net/http" "os" + "os/exec" "os/signal" "path" "path/filepath" @@ -28,10 +26,15 @@ import ( "syscall" "time" + "github.com/up9inc/mizu/shared/kubernetes" + v1 "k8s.io/api/core/v1" + + "github.com/antelman107/net-wait-go/wait" "github.com/gin-contrib/static" "github.com/gin-gonic/gin" "github.com/gorilla/websocket" "github.com/op/go-logging" + basenine "github.com/up9inc/basenine/client/go" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" "github.com/up9inc/mizu/tap" @@ -49,10 +52,12 @@ var harsDir = flag.String("hars-dir", "", "Directory to read hars from") var extensions []*tapApi.Extension // global var extensionsMap map[string]*tapApi.Extension // global +var startTime int64 + const ( - socketConnectionRetries = 10 + socketConnectionRetries = 10 socketConnectionRetryDelay = time.Second * 2 - socketHandshakeTimeout = time.Second * 2 + socketHandshakeTimeout = time.Second * 2 ) func main() { @@ -89,17 +94,17 @@ func main() { panic("API server address must be provided with --api-server-address when using --tap") } + hostMode := os.Getenv(shared.HostModeEnvVar) == "1" + tapOpts := &tap.TapOpts{HostMode: hostMode} tapTargets := getTapTargets() if tapTargets != nil { - tap.SetFilterAuthorities(tapTargets) - logger.Log.Infof("Filtering for the following authorities: %v", tap.GetFilterIPs()) + tapOpts.FilterAuthorities = tapTargets + logger.Log.Infof("Filtering for the following authorities: %v", tapOpts.FilterAuthorities) } filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem) filteringOptions := getTrafficFilteringOptions() - hostMode := os.Getenv(shared.HostModeEnvVar) == "1" - tapOpts := &tap.TapOpts{HostMode: hostMode} tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, extensions, filteringOptions) socketConnection, err := dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay) if err != nil { @@ -109,7 +114,8 @@ func main() { go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel) } else if *apiServerMode { - database.InitDataBase(config.Config.AgentDatabasePath) + startBasenineServer(shared.BasenineHost, shared.BaseninePort) + startTime = time.Now().UnixNano() / int64(time.Millisecond) api.StartResolving(*namespace) outputItemsChannel := make(chan *tapApi.OutputChannelItem) @@ -142,6 +148,53 @@ func main() { logger.Log.Info("Exiting") } +func startBasenineServer(host string, port string) { + cmd := exec.Command("basenine", "-addr", host, "-port", port, "-persistent") + cmd.Dir = config.Config.AgentDatabasePath + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Start() + if err != nil { + logger.Log.Panicf("Failed starting Basenine: %v", err) + } + + if !wait.New( + wait.WithProto("tcp"), + wait.WithWait(200*time.Millisecond), + wait.WithBreak(50*time.Millisecond), + wait.WithDeadline(5*time.Second), + wait.WithDebug(true), + ).Do([]string{fmt.Sprintf("%s:%s", host, port)}) { + logger.Log.Panicf("Basenine is not available: %v", err) + } + + // Make a channel to gracefully exit Basenine. + channel := make(chan os.Signal) + signal.Notify(channel, os.Interrupt, syscall.SIGTERM) + + // Handle the channel. + go func() { + <-channel + cmd.Process.Signal(syscall.SIGTERM) + }() + + // Limit the database size to default 200MB + err = basenine.Limit(host, port, config.Config.MaxDBSizeBytes) + if err != nil { + logger.Log.Panicf("Error while limiting database size: %v", err) + } + + for _, extension := range extensions { + macros := extension.Dissector.Macros() + for macro, expanded := range macros { + err = basenine.Macro(host, port, macro, expanded) + if err != nil { + logger.Log.Panicf("Error while adding a macro: %v", err) + } + } + } +} + func loadExtensions() { dir, _ := filepath.Abs(filepath.Dir(os.Args[0])) extensionsDir := path.Join(dir, "./extensions/") @@ -154,7 +207,7 @@ func loadExtensions() { extensionsMap = make(map[string]*tapApi.Extension) for i, file := range files { filename := file.Name() - logger.Log.Infof("Loading extension: %s\n", filename) + logger.Log.Infof("Loading extension: %s", filename) extension := &tapApi.Extension{ Path: path.Join(extensionsDir, filename), } @@ -166,7 +219,7 @@ func loadExtensions() { var ok bool dissector, ok = symDissector.(tapApi.Dissector) if err != nil || !ok { - panic(fmt.Sprintf("Failed to load the extension: %s\n", extension.Path)) + panic(fmt.Sprintf("Failed to load the extension: %s", extension.Path)) } dissector.Register(extension) extension.Dissector = dissector @@ -179,7 +232,7 @@ func loadExtensions() { }) for _, extension := range extensions { - logger.Log.Infof("Extension Properties: %+v\n", extension) + logger.Log.Infof("Extension Properties: %+v", extension) } controllers.InitExtensionsMap(extensionsMap) @@ -200,7 +253,8 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) { app.Use(static.ServeRoot("/", "./site")) app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before - api.WebSocketRoutes(app, &eventHandlers) + api.WebSocketRoutes(app, &eventHandlers, startTime) + routes.QueryRoutes(app) routes.EntriesRoutes(app) routes.MetadataRoutes(app) routes.StatusRoutes(app) @@ -210,7 +264,12 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - if _, err := startMizuTapperSyncer(ctx); err != nil { + kubernetesProvider, err := kubernetes.NewProviderInCluster() + if err != nil { + logger.Log.Fatalf("error creating k8s provider: %+v", err) + } + + if _, err := startMizuTapperSyncer(ctx, kubernetesProvider); err != nil { logger.Log.Fatalf("error initializing tapper syncer: %+v", err) } } @@ -245,8 +304,8 @@ func CORSMiddleware() gin.HandlerFunc { } } -func parseEnvVar(env string) map[string][]string { - var mapOfList map[string][]string +func parseEnvVar(env string) map[string][]v1.Pod { + var mapOfList map[string][]v1.Pod val, present := os.LookupEnv(env) @@ -256,12 +315,12 @@ func parseEnvVar(env string) map[string][]string { err := json.Unmarshal([]byte(val), &mapOfList) if err != nil { - panic(fmt.Sprintf("env var %s's value of %s is invalid! must be map[string][]string %v", env, mapOfList, err)) + panic(fmt.Sprintf("env var %s's value of %v is invalid! must be map[string][]v1.Pod %v", env, mapOfList, err)) } return mapOfList } -func getTapTargets() []string { +func getTapTargets() []v1.Pod { nodeName := os.Getenv(shared.NodeNameEnvVar) tappedAddressesPerNodeDict := parseEnvVar(shared.TappedAddressesPerNodeDictEnvVar) return tappedAddressesPerNodeDict[nodeName] @@ -344,10 +403,11 @@ func getSyncEntriesConfig() *shared.SyncEntriesConfig { } func determineLogLevel() (logLevel logging.Level) { - logLevel = logging.INFO - if os.Getenv(shared.DebugModeEnvVar) == "1" { - logLevel = logging.DEBUG + logLevel, err := logging.LogLevel(os.Getenv(shared.LogLevelEnvVar)) + if err != nil { + logLevel = logging.INFO } + return } @@ -361,7 +421,7 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time. socketConnection, _, err := dialer.Dial(socketAddress, nil) if err != nil { if i < retryAmount { - logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay / time.Second) + logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay/time.Second) time.Sleep(retryDelay) } } else { @@ -371,13 +431,7 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time. return nil, lastErr } - -func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, error){ - provider, err := kubernetes.NewProviderInCluster() - if err != nil { - return nil, err - } - +func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider) (*kubernetes.MizuTapperSyncer, error) { tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{ TargetNamespaces: config.Config.TargetNamespaces, PodFilterRegex: config.Config.TapTargetRegex.Regexp, @@ -385,11 +439,12 @@ func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, e AgentImage: config.Config.AgentImage, TapperResources: config.Config.TapperResources, ImagePullPolicy: v1.PullPolicy(config.Config.PullPolicy), - DumpLogs: config.Config.DumpLogs, + LogLevel: config.Config.LogLevel, IgnoredUserAgents: config.Config.IgnoredUserAgents, MizuApiFilteringOptions: config.Config.MizuApiFilteringOptions, MizuServiceAccountExists: true, //assume service account exists since daemon mode will not function without it anyway - }) + Istio: config.Config.Istio, + }, time.Now()) if err != nil { return nil, err @@ -405,19 +460,31 @@ func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, e return } logger.Log.Fatalf("fatal tap syncer error: %v", syncerErr) - case _, ok := <-tapperSyncer.TapPodChangesOut: + case tapPodChangeEvent, ok := <-tapperSyncer.TapPodChangesOut: if !ok { logger.Log.Debug("mizuTapperSyncer pod changes channel closed, ending listener loop") return } - tapStatus := shared.TapStatus{Pods: kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods)} + providers.TapStatus = shared.TapStatus{Pods: kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods)} - serializedTapStatus, err := json.Marshal(shared.CreateWebSocketStatusMessage(tapStatus)) + tappedPodsStatus := utils.GetTappedPodsStatus() + + serializedTapStatus, err := json.Marshal(shared.CreateWebSocketStatusMessage(tappedPodsStatus)) if err != nil { logger.Log.Fatalf("error serializing tap status: %v", err) } api.BroadcastToBrowserClients(serializedTapStatus) - providers.TapStatus.Pods = tapStatus.Pods + providers.ExpectedTapperAmount = tapPodChangeEvent.ExpectedTapperAmount + case tapperStatus, ok := <-tapperSyncer.TapperStatusChangedOut: + if !ok { + logger.Log.Debug("mizuTapperSyncer tapper status changed channel closed, ending listener loop") + return + } + if providers.TappersStatus == nil { + providers.TappersStatus = make(map[string]shared.TapperStatus) + } + providers.TappersStatus[tapperStatus.NodeName] = tapperStatus + case <-ctx.Done(): logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done") return diff --git a/agent/pkg/api/main.go b/agent/pkg/api/main.go index 56968ff3c..42f86817a 100644 --- a/agent/pkg/api/main.go +++ b/agent/pkg/api/main.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "mizuserver/pkg/database" "mizuserver/pkg/holder" "mizuserver/pkg/providers" "os" @@ -14,15 +13,16 @@ import ( "strings" "time" - "go.mongodb.org/mongo-driver/bson/primitive" - "github.com/google/martian/har" + "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" tapApi "github.com/up9inc/mizu/tap/api" "mizuserver/pkg/models" "mizuserver/pkg/resolver" "mizuserver/pkg/utils" + + basenine "github.com/up9inc/basenine/client/go" ) var k8sResolver *resolver.Resolver @@ -76,7 +76,7 @@ func startReadingFiles(workingDir string) { sort.Sort(utils.ByModTime(harFiles)) if len(harFiles) == 0 { - logger.Log.Infof("Waiting for new files\n") + logger.Log.Infof("Waiting for new files") time.Sleep(3 * time.Second) continue } @@ -99,11 +99,17 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension panic("Channel of captured messages is nil") } + connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort) + if err != nil { + panic(err) + } + connection.InsertMode() + disableOASValidation := false ctx := context.Background() doc, contractContent, router, err := loadOAS(ctx) if err != nil { - logger.Log.Infof("Disabled OAS validation: %s\n", err.Error()) + logger.Log.Infof("Disabled OAS validation: %s", err.Error()) disableOASValidation = true } @@ -112,13 +118,13 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension extension := extensionsMap[item.Protocol.Name] resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo) - mizuEntry := extension.Dissector.Analyze(item, primitive.NewObjectID().Hex(), resolvedSource, resolvedDestionation) + mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation) baseEntry := extension.Dissector.Summarize(mizuEntry) - mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry) + mizuEntry.Base = baseEntry if extension.Protocol.Name == "http" { if !disableOASValidation { var httpPair tapApi.HTTPRequestResponsePair - json.Unmarshal([]byte(mizuEntry.Entry), &httpPair) + json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair) contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent) baseEntry.ContractStatus = contract.Status @@ -128,18 +134,18 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension mizuEntry.ContractContent = contract.Content } - var pair tapApi.RequestResponsePair - json.Unmarshal([]byte(mizuEntry.Entry), &pair) - harEntry, err := utils.NewEntry(&pair) + harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime) if err == nil { - rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Service) + rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name) baseEntry.Rules = rules } } - database.CreateEntry(mizuEntry) - baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(baseEntry) - BroadcastToBrowserClients(baseEntryBytes) + data, err := json.Marshal(mizuEntry) + if err != nil { + panic(err) + } + connection.SendText(string(data)) } } @@ -148,7 +154,7 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re unresolvedSource := connectionInfo.ClientIP resolvedSource = k8sResolver.Resolve(unresolvedSource) if resolvedSource == "" { - logger.Log.Debugf("Cannot find resolved name to source: %s\n", unresolvedSource) + logger.Log.Debugf("Cannot find resolved name to source: %s", unresolvedSource) if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" { return } @@ -156,7 +162,7 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort) resolvedDestination = k8sResolver.Resolve(unresolvedDestination) if resolvedDestination == "" { - logger.Log.Debugf("Cannot find resolved name to dest: %s\n", unresolvedDestination) + logger.Log.Debugf("Cannot find resolved name to dest: %s", unresolvedDestination) if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" { return } @@ -171,21 +177,3 @@ func CheckIsServiceIP(address string) bool { } return k8sResolver.CheckIsServiceIP(address) } - -// gives a rough estimate of the size this will take up in the db, good enough for maintaining db size limit accurately -func getEstimatedEntrySizeBytes(mizuEntry *tapApi.MizuEntry) int { - sizeBytes := len(mizuEntry.Entry) - sizeBytes += len(mizuEntry.EntryId) - sizeBytes += len(mizuEntry.Service) - sizeBytes += len(mizuEntry.Url) - sizeBytes += len(mizuEntry.Method) - sizeBytes += len(mizuEntry.RequestSenderIp) - sizeBytes += len(mizuEntry.ResolvedDestination) - sizeBytes += len(mizuEntry.ResolvedSource) - sizeBytes += 8 // Status bytes (sqlite integer is always 8 bytes) - sizeBytes += 8 // Timestamp bytes - sizeBytes += 8 // SizeBytes bytes - sizeBytes += 1 // IsOutgoing bytes - - return sizeBytes -} diff --git a/agent/pkg/api/socket_routes.go b/agent/pkg/api/socket_routes.go index 7e765268c..a96bb11ff 100644 --- a/agent/pkg/api/socket_routes.go +++ b/agent/pkg/api/socket_routes.go @@ -1,13 +1,18 @@ package api import ( + "encoding/json" "errors" + "fmt" + "mizuserver/pkg/models" "net/http" "sync" "time" "github.com/gin-gonic/gin" "github.com/gorilla/websocket" + basenine "github.com/up9inc/basenine/client/go" + "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/debounce" "github.com/up9inc/mizu/shared/logger" ) @@ -39,17 +44,17 @@ func init() { connectedWebsockets = make(map[int]*SocketConnection, 0) } -func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) { +func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) { app.GET("/ws", func(c *gin.Context) { - websocketHandler(c.Writer, c.Request, eventHandlers, false) + websocketHandler(c.Writer, c.Request, eventHandlers, false, startTime) }) app.GET("/wsTapper", func(c *gin.Context) { - websocketHandler(c.Writer, c.Request, eventHandlers, true) + websocketHandler(c.Writer, c.Request, eventHandlers, true, startTime) }) } -func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool) { - conn, err := websocketUpgrader.Upgrade(w, r, nil) +func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool, startTime int64) { + ws, err := websocketUpgrader.Upgrade(w, r, nil) if err != nil { logger.Log.Errorf("Failed to set websocket upgrade: %v", err) return @@ -59,30 +64,117 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even connectedWebsocketIdCounter++ socketId := connectedWebsocketIdCounter - connectedWebsockets[socketId] = &SocketConnection{connection: conn, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper} + connectedWebsockets[socketId] = &SocketConnection{connection: ws, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper} websocketIdsLock.Unlock() + var connection *basenine.Connection + var isQuerySet bool + + // `!isTapper` means it's a connection from the web UI + if !isTapper { + connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort) + if err != nil { + panic(err) + } + } + + data := make(chan []byte) + meta := make(chan []byte) + defer func() { + data <- []byte(basenine.CloseChannel) + meta <- []byte(basenine.CloseChannel) + connection.Close() socketCleanup(socketId, connectedWebsockets[socketId]) }() eventHandlers.WebSocketConnect(socketId, isTapper) + startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(startTime) + SendToSocket(socketId, startTimeBytes) + for { - _, msg, err := conn.ReadMessage() + _, msg, err := ws.ReadMessage() if err != nil { logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err) break } - eventHandlers.WebSocketMessage(socketId, msg) + + if !isTapper && !isQuerySet { + query := string(msg) + err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query) + if err != nil { + toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{ + Type: "error", + AutoClose: 5000, + Text: fmt.Sprintf("Syntax error: %s", err.Error()), + }) + SendToSocket(socketId, toastBytes) + break + } + + isQuerySet = true + + handleDataChannel := func(c *basenine.Connection, data chan []byte) { + for { + bytes := <-data + + if string(bytes) == basenine.CloseChannel { + return + } + + var dataMap map[string]interface{} + err = json.Unmarshal(bytes, &dataMap) + + var base map[string]interface{} + switch dataMap["base"].(type) { + case map[string]interface{}: + base = dataMap["base"].(map[string]interface{}) + base["id"] = uint(dataMap["id"].(float64)) + default: + logger.Log.Debugf("Base field has an unrecognized type: %+v", dataMap) + continue + } + + baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base) + SendToSocket(socketId, baseEntryBytes) + } + } + + handleMetaChannel := func(c *basenine.Connection, meta chan []byte) { + for { + bytes := <-meta + + if string(bytes) == basenine.CloseChannel { + return + } + + var metadata *basenine.Metadata + err = json.Unmarshal(bytes, &metadata) + if err != nil { + logger.Log.Debugf("Error recieving metadata: %v", err.Error()) + } + + metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata) + SendToSocket(socketId, metadataBytes) + } + } + + go handleDataChannel(connection, data) + go handleMetaChannel(connection, meta) + + connection.Query(query, data, meta) + } else { + eventHandlers.WebSocketMessage(socketId, msg) + } } } func socketCleanup(socketId int, socketConnection *SocketConnection) { err := socketConnection.connection.Close() if err != nil { - logger.Log.Errorf("Error closing socket connection for socket id %d: %v\n", socketId, err) + logger.Log.Errorf("Error closing socket connection for socket id %d: %v", socketId, err) } websocketIdsLock.Lock() diff --git a/agent/pkg/api/socket_server_handlers.go b/agent/pkg/api/socket_server_handlers.go index 9a6c8d3e8..a02aaa82b 100644 --- a/agent/pkg/api/socket_server_handlers.go +++ b/agent/pkg/api/socket_server_handlers.go @@ -65,14 +65,14 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) { var socketMessageBase shared.WebSocketMessageMetadata err := json.Unmarshal(message, &socketMessageBase) if err != nil { - logger.Log.Infof("Could not unmarshal websocket message %v\n", err) + logger.Log.Infof("Could not unmarshal websocket message %v", err) } else { switch socketMessageBase.MessageType { case shared.WebSocketMessageTypeTappedEntry: var tappedEntryMessage models.WebSocketTappedEntryMessage err := json.Unmarshal(message, &tappedEntryMessage) if err != nil { - logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err) + logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err) } else { // NOTE: This is where the message comes back from the intermediate WebSocket to code. h.SocketOutChannel <- tappedEntryMessage.Data @@ -81,16 +81,15 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) { var statusMessage shared.WebSocketStatusMessage err := json.Unmarshal(message, &statusMessage) if err != nil { - logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err) + logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err) } else { - providers.TapStatus.Pods = statusMessage.TappingStatus.Pods BroadcastToBrowserClients(message) } case shared.WebsocketMessageTypeOutboundLink: var outboundLinkMessage models.WebsocketOutboundLinkMessage err := json.Unmarshal(message, &outboundLinkMessage) if err != nil { - logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err) + logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err) } else { handleTLSLink(outboundLinkMessage) } diff --git a/agent/pkg/controllers/entries_controller.go b/agent/pkg/controllers/entries_controller.go index 32422d5e4..0b2f0724a 100644 --- a/agent/pkg/controllers/entries_controller.go +++ b/agent/pkg/controllers/entries_controller.go @@ -2,14 +2,19 @@ package controllers import ( "encoding/json" - "fmt" - "github.com/gin-gonic/gin" - tapApi "github.com/up9inc/mizu/tap/api" - "mizuserver/pkg/database" "mizuserver/pkg/models" "mizuserver/pkg/utils" "mizuserver/pkg/validation" "net/http" + "strconv" + "time" + + "github.com/gin-gonic/gin" + + basenine "github.com/up9inc/basenine/client/go" + "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/shared/logger" + tapApi "github.com/up9inc/mizu/tap/api" ) var extensionsMap map[string]*tapApi.Extension // global @@ -18,78 +23,113 @@ func InitExtensionsMap(ref map[string]*tapApi.Extension) { extensionsMap = ref } -func GetEntries(c *gin.Context) { - entriesFilter := &models.EntriesFilter{} - - if err := c.BindQuery(entriesFilter); err != nil { - c.JSON(http.StatusBadRequest, err) - } - err := validation.Validate(entriesFilter) +func Error(c *gin.Context, err error) bool { if err != nil { + logger.Log.Errorf("Error getting entry: %v", err) + c.Error(err) + c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ + "error": true, + "type": "error", + "autoClose": "5000", + "msg": err.Error(), + }) + return true // signal that there was an error and the caller should return + } + return false // no error, can continue +} + +func GetEntries(c *gin.Context) { + entriesRequest := &models.EntriesRequest{} + + if err := c.BindQuery(entriesRequest); err != nil { c.JSON(http.StatusBadRequest, err) } - - order := database.OperatorToOrderMapping[entriesFilter.Operator] - operatorSymbol := database.OperatorToSymbolMapping[entriesFilter.Operator] - var entries []tapApi.MizuEntry - database.GetEntriesTable(). - Order(fmt.Sprintf("timestamp %s", order)). - Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)). - Limit(entriesFilter.Limit). - Find(&entries) - - if len(entries) > 0 && order == database.OrderDesc { - // the entries always order from oldest to newest - we should reverse - utils.ReverseSlice(entries) + validationError := validation.Validate(entriesRequest) + if validationError != nil { + c.JSON(http.StatusBadRequest, validationError) } - baseEntries := make([]tapApi.BaseEntryDetails, 0) - for _, entry := range entries { - baseEntryDetails := tapApi.BaseEntryDetails{} - if err := models.GetEntry(&entry, &baseEntryDetails); err != nil { - continue - } - - var pair tapApi.RequestResponsePair - json.Unmarshal([]byte(entry.Entry), &pair) - harEntry, err := utils.NewEntry(&pair) - if err == nil { - rules, _, _ := models.RunValidationRulesState(*harEntry, entry.Service) - baseEntryDetails.Rules = rules - } - - baseEntries = append(baseEntries, baseEntryDetails) + if entriesRequest.TimeoutMs == 0 { + entriesRequest.TimeoutMs = 3000 } - c.JSON(http.StatusOK, baseEntries) + data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort, + entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query, + entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond) + if err != nil { + c.JSON(http.StatusInternalServerError, validationError) + } + + response := &models.EntriesResponse{} + var dataSlice []interface{} + + for _, row := range data { + var dataMap map[string]interface{} + err = json.Unmarshal(row, &dataMap) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{ + "error": true, + "type": "error", + "autoClose": "5000", + "msg": string(row), + }) + return // exit + } + + base := dataMap["base"].(map[string]interface{}) + base["id"] = uint(dataMap["id"].(float64)) + + dataSlice = append(dataSlice, base) + } + + var metadata *basenine.Metadata + err = json.Unmarshal(meta, &metadata) + if err != nil { + logger.Log.Debugf("Error recieving metadata: %v", err.Error()) + } + + response.Data = dataSlice + response.Meta = metadata + + c.JSON(http.StatusOK, response) } func GetEntry(c *gin.Context) { - var entryData tapApi.MizuEntry - database.GetEntriesTable(). - Where(map[string]string{"entryId": c.Param("entryId")}). - First(&entryData) + id, _ := strconv.Atoi(c.Param("id")) + var entry tapApi.MizuEntry + bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id) + if Error(c, err) { + return // exit + } + err = json.Unmarshal(bytes, &entry) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{ + "error": true, + "type": "error", + "autoClose": "5000", + "msg": string(bytes), + }) + return // exit + } - extension := extensionsMap[entryData.ProtocolName] - protocol, representation, bodySize, _ := extension.Dissector.Represent(&entryData) + extension := extensionsMap[entry.Protocol.Name] + representation, bodySize, _ := extension.Dissector.Represent(entry.Request, entry.Response) var rules []map[string]interface{} var isRulesEnabled bool - if entryData.ProtocolName == "http" { - var pair tapApi.RequestResponsePair - json.Unmarshal([]byte(entryData.Entry), &pair) - harEntry, _ := utils.NewEntry(&pair) - _, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entryData.Service) + if entry.Protocol.Name == "http" { + harEntry, _ := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime) + _, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name) isRulesEnabled = _isRulesEnabled inrec, _ := json.Marshal(rulesMatched) json.Unmarshal(inrec, &rules) } c.JSON(http.StatusOK, tapApi.MizuEntryWrapper{ - Protocol: protocol, + Protocol: entry.Protocol, Representation: string(representation), BodySize: bodySize, - Data: entryData, + Data: entry, Rules: rules, IsRulesEnabled: isRulesEnabled, }) diff --git a/agent/pkg/controllers/query_controller.go b/agent/pkg/controllers/query_controller.go new file mode 100644 index 000000000..f74230f8b --- /dev/null +++ b/agent/pkg/controllers/query_controller.go @@ -0,0 +1,31 @@ +package controllers + +import ( + "net/http" + + "github.com/gin-gonic/gin" + basenine "github.com/up9inc/basenine/client/go" + "github.com/up9inc/mizu/shared" +) + +type ValidateResponse struct { + Valid bool `json:"valid"` + Message string `json:"message"` +} + +func PostValidate(c *gin.Context) { + query := c.PostForm("query") + valid := true + message := "" + + err := basenine.Validate(shared.BasenineHost, shared.BaseninePort, query) + if err != nil { + valid = false + message = err.Error() + } + + c.JSON(http.StatusOK, ValidateResponse{ + Valid: valid, + Message: message, + }) +} diff --git a/agent/pkg/controllers/status_controller.go b/agent/pkg/controllers/status_controller.go index fab3bb690..0987ddc16 100644 --- a/agent/pkg/controllers/status_controller.go +++ b/agent/pkg/controllers/status_controller.go @@ -2,27 +2,41 @@ package controllers import ( "encoding/json" - "mizuserver/pkg/api" - "mizuserver/pkg/holder" - "mizuserver/pkg/providers" - "mizuserver/pkg/up9" - "mizuserver/pkg/validation" - "net/http" - + "fmt" "github.com/gin-gonic/gin" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" + "mizuserver/pkg/api" + "mizuserver/pkg/config" + "mizuserver/pkg/holder" + "mizuserver/pkg/providers" + "mizuserver/pkg/up9" + "mizuserver/pkg/utils" + "mizuserver/pkg/validation" + "net/http" ) func HealthCheck(c *gin.Context) { + if config.Config.DaemonMode { + if providers.ExpectedTapperAmount != providers.TappersCount { + c.JSON(http.StatusInternalServerError, fmt.Sprintf("expecting more tappers than are actually connected (%d expected, %d connected)", providers.ExpectedTapperAmount, providers.TappersCount)) + return + } + } + + tappers := make([]shared.TapperStatus, 0) + for _, value := range providers.TappersStatus { + tappers = append(tappers, value) + } + response := shared.HealthResponse{ - TapStatus: providers.TapStatus, - TappersCount: providers.TappersCount, + TapStatus: providers.TapStatus, + TappersCount: providers.TappersCount, + TappersStatus: tappers, } c.JSON(http.StatusOK, response) } - func PostTappedPods(c *gin.Context) { tapStatus := &shared.TapStatus{} if err := c.Bind(tapStatus); err != nil { @@ -35,14 +49,38 @@ func PostTappedPods(c *gin.Context) { } logger.Log.Infof("[Status] POST request: %d tapped pods", len(tapStatus.Pods)) providers.TapStatus.Pods = tapStatus.Pods - message := shared.CreateWebSocketStatusMessage(*tapStatus) + broadcastTappedPodsStatus() +} + +func broadcastTappedPodsStatus() { + tappedPodsStatus := utils.GetTappedPodsStatus() + + message := shared.CreateWebSocketStatusMessage(tappedPodsStatus) if jsonBytes, err := json.Marshal(message); err != nil { - logger.Log.Errorf("Could not Marshal message %v\n", err) + logger.Log.Errorf("Could not Marshal message %v", err) } else { api.BroadcastToBrowserClients(jsonBytes) } } +func PostTapperStatus(c *gin.Context) { + tapperStatus := &shared.TapperStatus{} + if err := c.Bind(tapperStatus); err != nil { + c.JSON(http.StatusBadRequest, err) + return + } + if err := validation.Validate(tapperStatus); err != nil { + c.JSON(http.StatusBadRequest, err) + return + } + logger.Log.Infof("[Status] POST request, tapper status: %v", tapperStatus) + if providers.TappersStatus == nil { + providers.TappersStatus = make(map[string]shared.TapperStatus) + } + providers.TappersStatus[tapperStatus.NodeName] = *tapperStatus + broadcastTappedPodsStatus() +} + func GetTappersCount(c *gin.Context) { c.JSON(http.StatusOK, providers.TappersCount) } @@ -58,7 +96,8 @@ func GetAuthStatus(c *gin.Context) { } func GetTappingStatus(c *gin.Context) { - c.JSON(http.StatusOK, providers.TapStatus) + tappedPodsStatus := utils.GetTappedPodsStatus() + c.JSON(http.StatusOK, tappedPodsStatus) } func AnalyzeInformation(c *gin.Context) { diff --git a/agent/pkg/database/main.go b/agent/pkg/database/main.go deleted file mode 100644 index 6d199823c..000000000 --- a/agent/pkg/database/main.go +++ /dev/null @@ -1,78 +0,0 @@ -package database - -import ( - "fmt" - "mizuserver/pkg/utils" - "time" - - "gorm.io/driver/sqlite" - "gorm.io/gorm" - "gorm.io/gorm/logger" - - tapApi "github.com/up9inc/mizu/tap/api" -) - -const ( - OrderDesc = "desc" - OrderAsc = "asc" - LT = "lt" - GT = "gt" - TimeFormat = "2006-01-02 15:04:05.000000000" -) - -var ( - DB *gorm.DB - IsDBLocked = false - OperatorToSymbolMapping = map[string]string{ - LT: "<", - GT: ">", - } - OperatorToOrderMapping = map[string]string{ - LT: OrderDesc, - GT: OrderAsc, - } -) - -var DBPath string - -func GetEntriesTable() *gorm.DB { - return DB.Table("mizu_entries") -} - -func CreateEntry(entry *tapApi.MizuEntry) { - if IsDBLocked { - return - } - GetEntriesTable().Create(entry) -} - -func InitDataBase(databasePath string) *gorm.DB { - DBPath = databasePath - DB, _ = gorm.Open(sqlite.Open(databasePath), &gorm.Config{ - Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond}, - }) - _ = DB.AutoMigrate(&tapApi.MizuEntry{}) // this will ensure table is created - go StartEnforcingDatabaseSize() - return DB -} - -func GetEntriesFromDb(timeFrom time.Time, timeTo time.Time, protocolName *string) []tapApi.MizuEntry { - order := OrderDesc - protocolNameCondition := "1 = 1" - if protocolName != nil { - protocolNameCondition = fmt.Sprintf("protocolName = '%s'", *protocolName) - } - - var entries []tapApi.MizuEntry - GetEntriesTable(). - Where(protocolNameCondition). - Where(fmt.Sprintf("created_at BETWEEN '%s' AND '%s'", timeFrom.Format(TimeFormat), timeTo.Format(TimeFormat))). - Order(fmt.Sprintf("timestamp %s", order)). - Find(&entries) - - if len(entries) > 0 { - // the entries always order from oldest to newest so we should revers - utils.ReverseSlice(entries) - } - return entries -} diff --git a/agent/pkg/database/size_enforcer.go b/agent/pkg/database/size_enforcer.go deleted file mode 100644 index 7e06da7fe..000000000 --- a/agent/pkg/database/size_enforcer.go +++ /dev/null @@ -1,102 +0,0 @@ -package database - -import ( - "mizuserver/pkg/config" - "os" - "time" - - "github.com/fsnotify/fsnotify" - "github.com/up9inc/mizu/shared/debounce" - "github.com/up9inc/mizu/shared/logger" - "github.com/up9inc/mizu/shared/units" - tapApi "github.com/up9inc/mizu/tap/api" -) - -const percentageOfMaxSizeBytesToPrune = 15 - -func StartEnforcingDatabaseSize() { - watcher, err := fsnotify.NewWatcher() - if err != nil { - logger.Log.Fatalf("Error creating filesystem watcher for db size enforcement: %v\n", err) - return - } - - checkFileSizeDebouncer := debounce.NewDebouncer(5*time.Second, func() { - checkFileSize(config.Config.MaxDBSizeBytes) - }) - - go func() { - for { - select { - case event, ok := <-watcher.Events: - if !ok { - return // closed channel - } - if event.Op == fsnotify.Write { - checkFileSizeDebouncer.SetOn() - } - case err, ok := <-watcher.Errors: - if !ok { - return // closed channel - } - logger.Log.Errorf("filesystem watcher encountered error:%v", err) - } - } - }() - - err = watcher.Add(DBPath) - if err != nil { - logger.Log.Fatalf("Error adding %s to filesystem watcher for db size enforcement: %v\n", DBPath, err) - } -} - -func checkFileSize(maxSizeBytes int64) { - fileStat, err := os.Stat(DBPath) - if err != nil { - logger.Log.Errorf("Error checking %s file size: %v", DBPath, err) - } else { - if fileStat.Size() > maxSizeBytes { - pruneOldEntries(fileStat.Size()) - } - } -} - -func pruneOldEntries(currentFileSize int64) { - // sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here - IsDBLocked = true - defer func() { IsDBLocked = false }() - - amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune) - - rows, err := GetEntriesTable().Limit(10000).Order("id").Rows() - if err != nil { - logger.Log.Errorf("Error getting 10000 first db rows: %v", err) - return - } - - entryIdsToRemove := make([]uint, 0) - bytesToBeRemoved := int64(0) - for rows.Next() { - if bytesToBeRemoved >= amountOfBytesToTrim { - break - } - var entry tapApi.MizuEntry - err = DB.ScanRows(rows, &entry) - if err != nil { - logger.Log.Errorf("Error scanning db row: %v", err) - continue - } - - entryIdsToRemove = append(entryIdsToRemove, entry.ID) - bytesToBeRemoved += int64(entry.EstimatedSizeBytes) - } - - if len(entryIdsToRemove) > 0 { - GetEntriesTable().Where(entryIdsToRemove).Delete(tapApi.MizuEntry{}) - // VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this - DB.Exec("VACUUM") - logger.Log.Errorf("Removed %d rows and cleared %s", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved)) - } else { - logger.Log.Error("Found no rows to remove when pruning") - } -} diff --git a/agent/pkg/models/models.go b/agent/pkg/models/models.go index 0209f3675..84f8e9842 100644 --- a/agent/pkg/models/models.go +++ b/agent/pkg/models/models.go @@ -2,12 +2,12 @@ package models import ( "encoding/json" + "mizuserver/pkg/rules" tapApi "github.com/up9inc/mizu/tap/api" - "mizuserver/pkg/rules" - "github.com/google/martian/har" + basenine "github.com/up9inc/basenine/client/go" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/tap" ) @@ -16,15 +16,22 @@ func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error { return v.UnmarshalData(r) } -type EntriesFilter struct { - Limit int `form:"limit" validate:"required,min=1,max=200"` - Operator string `form:"operator" validate:"required,oneof='lt' 'gt'"` - Timestamp int64 `form:"timestamp" validate:"required,min=1"` +type EntriesRequest struct { + LeftOff int `form:"leftOff" validate:"required,min=-1"` + Direction int `form:"direction" validate:"required,oneof='1' '-1'"` + Query string `form:"query"` + Limit int `form:"limit" validate:"required,min=1"` + TimeoutMs int `form:"timeoutMs" validate:"min=1"` +} + +type EntriesResponse struct { + Data []interface{} `json:"data"` + Meta *basenine.Metadata `json:"meta"` } type WebSocketEntryMessage struct { *shared.WebSocketMessageMetadata - Data *tapApi.BaseEntryDetails `json:"data,omitempty"` + Data map[string]interface{} `json:"data,omitempty"` } type WebSocketTappedEntryMessage struct { @@ -42,7 +49,28 @@ type AuthStatus struct { Model string `json:"model"` } -func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntryDetails) ([]byte, error) { +type ToastMessage struct { + Type string `json:"type"` + AutoClose uint `json:"autoClose"` + Text string `json:"text"` +} + +type WebSocketToastMessage struct { + *shared.WebSocketMessageMetadata + Data *ToastMessage `json:"data,omitempty"` +} + +type WebSocketQueryMetadataMessage struct { + *shared.WebSocketMessageMetadata + Data *basenine.Metadata `json:"data,omitempty"` +} + +type WebSocketStartTimeMessage struct { + *shared.WebSocketMessageMetadata + Data int64 `json:"data"` +} + +func CreateBaseEntryWebSocketMessage(base map[string]interface{}) ([]byte, error) { message := &WebSocketEntryMessage{ WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{ MessageType: shared.WebSocketMessageTypeEntry, @@ -72,6 +100,36 @@ func CreateWebsocketOutboundLinkMessage(base *tap.OutboundLink) ([]byte, error) return json.Marshal(message) } +func CreateWebsocketToastMessage(base *ToastMessage) ([]byte, error) { + message := &WebSocketToastMessage{ + WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{ + MessageType: shared.WebSocketMessageTypeToast, + }, + Data: base, + } + return json.Marshal(message) +} + +func CreateWebsocketQueryMetadataMessage(base *basenine.Metadata) ([]byte, error) { + message := &WebSocketQueryMetadataMessage{ + WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{ + MessageType: shared.WebSocketMessageTypeQueryMetadata, + }, + Data: base, + } + return json.Marshal(message) +} + +func CreateWebsocketStartTimeMessage(base int64) ([]byte, error) { + message := &WebSocketStartTimeMessage{ + WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{ + MessageType: shared.WebSocketMessageTypeStartTime, + }, + Data: base, + } + return json.Marshal(message) +} + // ExtendedHAR is the top level object of a HAR log. type ExtendedHAR struct { Log *ExtendedLog `json:"log"` diff --git a/agent/pkg/providers/stats_provider_test.go b/agent/pkg/providers/stats_provider_test.go index 4e0d3ff62..13acfece9 100644 --- a/agent/pkg/providers/stats_provider_test.go +++ b/agent/pkg/providers/stats_provider_test.go @@ -2,16 +2,10 @@ package providers_test import ( "fmt" - "mizuserver/pkg/config" - "mizuserver/pkg/database" "mizuserver/pkg/providers" "testing" ) -func init() { - database.InitDataBase(config.DefaultDatabasePath) -} - func TestNoEntryAddedCount(t *testing.T) { entriesStats := providers.GetGeneralStats() diff --git a/agent/pkg/providers/status_provider.go b/agent/pkg/providers/status_provider.go index 15928b9bf..85fe8793b 100644 --- a/agent/pkg/providers/status_provider.go +++ b/agent/pkg/providers/status_provider.go @@ -15,12 +15,13 @@ import ( const tlsLinkRetainmentTime = time.Minute * 15 var ( - TappersCount int - TapStatus shared.TapStatus - authStatus *models.AuthStatus - RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime) - - tappersCountLock = sync.Mutex{} + TappersCount int + TapStatus shared.TapStatus + TappersStatus map[string]shared.TapperStatus + authStatus *models.AuthStatus + RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime) + ExpectedTapperAmount = -1 //only relevant in daemon mode as cli manages tappers otherwise + tappersCountLock = sync.Mutex{} ) func GetAuthStatus() (*models.AuthStatus, error) { diff --git a/agent/pkg/resolver/resolver.go b/agent/pkg/resolver/resolver.go index 3d2ac5544..60533704d 100644 --- a/agent/pkg/resolver/resolver.go +++ b/agent/pkg/resolver/resolver.go @@ -164,10 +164,10 @@ func (resolver *Resolver) watchServices(ctx context.Context) error { func (resolver *Resolver) saveResolvedName(key string, resolved string, eventType watch.EventType) { if eventType == watch.Deleted { resolver.nameMap.Remove(key) - logger.Log.Infof("setting %s=nil\n", key) + logger.Log.Infof("setting %s=nil", key) } else { resolver.nameMap.Set(key, resolved) - logger.Log.Infof("setting %s=%s\n", key, resolved) + logger.Log.Infof("setting %s=%s", key, resolved) } } @@ -188,7 +188,7 @@ func (resolver *Resolver) infiniteErrorHandleRetryFunc(ctx context.Context, fun var statusError *k8serrors.StatusError if errors.As(err, &statusError) { if statusError.ErrStatus.Reason == metav1.StatusReasonForbidden { - logger.Log.Infof("Resolver loop encountered permission error, aborting event listening - %v\n", err) + logger.Log.Infof("Resolver loop encountered permission error, aborting event listening - %v", err) return } } diff --git a/agent/pkg/routes/entries_routes.go b/agent/pkg/routes/entries_routes.go index 8e578fea4..d931196a9 100644 --- a/agent/pkg/routes/entries_routes.go +++ b/agent/pkg/routes/entries_routes.go @@ -1,14 +1,15 @@ package routes import ( - "github.com/gin-gonic/gin" "mizuserver/pkg/controllers" + + "github.com/gin-gonic/gin" ) // EntriesRoutes defines the group of har entries routes. func EntriesRoutes(ginApp *gin.Engine) { routeGroup := ginApp.Group("/entries") - routeGroup.GET("/", controllers.GetEntries) // get entries (base/thin entries) - routeGroup.GET("/:entryId", controllers.GetEntry) // get single (full) entry + routeGroup.GET("/", controllers.GetEntries) // get entries (base/thin entries) and metadata + routeGroup.GET("/:id", controllers.GetEntry) // get single (full) entry } diff --git a/agent/pkg/routes/query_routes.go b/agent/pkg/routes/query_routes.go new file mode 100644 index 000000000..2807bd0b6 --- /dev/null +++ b/agent/pkg/routes/query_routes.go @@ -0,0 +1,13 @@ +package routes + +import ( + "mizuserver/pkg/controllers" + + "github.com/gin-gonic/gin" +) + +func QueryRoutes(ginApp *gin.Engine) { + routeGroup := ginApp.Group("/query") + + routeGroup.POST("/validate", controllers.PostValidate) +} diff --git a/agent/pkg/routes/status_routes.go b/agent/pkg/routes/status_routes.go index 505e0b5f5..dd601213d 100644 --- a/agent/pkg/routes/status_routes.go +++ b/agent/pkg/routes/status_routes.go @@ -11,6 +11,7 @@ func StatusRoutes(ginApp *gin.Engine) { routeGroup.GET("/health", controllers.HealthCheck) routeGroup.POST("/tappedPods", controllers.PostTappedPods) + routeGroup.POST("/tapperStatus", controllers.PostTapperStatus) routeGroup.GET("/tappersCount", controllers.GetTappersCount) routeGroup.GET("/tap", controllers.GetTappingStatus) diff --git a/agent/pkg/sensitiveDataFiltering/consts.go b/agent/pkg/sensitiveDataFiltering/consts.go deleted file mode 100644 index e5624de73..000000000 --- a/agent/pkg/sensitiveDataFiltering/consts.go +++ /dev/null @@ -1,10 +0,0 @@ -package sensitiveDataFiltering - -const maskedFieldPlaceholderValue = "[REDACTED]" - -//these values MUST be all lower case and contain no `-` or `_` characters -var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password", - "username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt", - "bearer", "clientid", "clientsecret", "redirecturi", "phonenumber", - "zip", "zipcode", "address", "country", "firstname", "lastname", - "middlename", "fname", "lname", "birthdate"} diff --git a/agent/pkg/up9/main.go b/agent/pkg/up9/main.go index 6fa9304bd..e44c2b9e2 100644 --- a/agent/pkg/up9/main.go +++ b/agent/pkg/up9/main.go @@ -7,15 +7,16 @@ import ( "encoding/json" "fmt" "io/ioutil" - "mizuserver/pkg/database" "mizuserver/pkg/utils" "net/http" "net/url" "regexp" "strings" + "sync" "time" "github.com/google/martian/har" + basenine "github.com/up9inc/basenine/client/go" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" tapApi "github.com/up9inc/mizu/tap/api" @@ -23,6 +24,7 @@ import ( const ( AnalyzeCheckSleepTime = 5 * time.Second + SentCountLogInterval = 100 ) type GuestToken struct { @@ -110,14 +112,14 @@ func GetAnalyzeInfo() *shared.AnalyzeStatus { } func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error { - logger.Log.Infof("Sync entries - started\n") + logger.Log.Infof("Sync entries - started") var ( token, model string guestMode bool ) if syncEntriesConfig.Token == "" { - logger.Log.Infof("Sync entries - creating anonymous token. env %s\n", syncEntriesConfig.Env) + logger.Log.Infof("Sync entries - creating anonymous token. env %s", syncEntriesConfig.Env) guestToken, err := createAnonymousToken(syncEntriesConfig.Env) if err != nil { return fmt.Errorf("failed creating anonymous token, err: %v", err) @@ -131,7 +133,7 @@ func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error { model = syncEntriesConfig.Workspace guestMode = false - logger.Log.Infof("Sync entries - upserting model. env %s, model %s\n", syncEntriesConfig.Env, model) + logger.Log.Infof("Sync entries - upserting model. env %s, model %s", syncEntriesConfig.Env, model) if err := upsertModel(token, model, syncEntriesConfig.Env); err != nil { return fmt.Errorf("failed upserting model, err: %v", err) } @@ -142,7 +144,7 @@ func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error { return fmt.Errorf("invalid model name, model name: %s", model) } - logger.Log.Infof("Sync entries - syncing. token: %s, model: %s, guest mode: %v\n", token, model, guestMode) + logger.Log.Infof("Sync entries - syncing. token: %s, model: %s, guest mode: %v", token, model, guestMode) go syncEntriesImpl(token, model, syncEntriesConfig.Env, syncEntriesConfig.UploadIntervalSec, guestMode) return nil @@ -204,51 +206,80 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva analyzeInformation.AnalyzeDestination = envPrefix analyzeInformation.SentCount = 0 - sleepTime := time.Second * time.Duration(uploadIntervalSec) + // "http or grpc" filter indicates that we're only interested in HTTP and gRPC entries + query := "http or grpc" - var timeFrom time.Time - protocolFilter := "http" + logger.Log.Infof("Getting entries from the database") - for { - timeTo := time.Now() - logger.Log.Infof("Getting entries from %v, to %v\n", timeFrom.Format(time.RFC3339Nano), timeTo.Format(time.RFC3339Nano)) - entriesArray := database.GetEntriesFromDb(timeFrom, timeTo, &protocolFilter) + var connection *basenine.Connection + var err error + connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort) + if err != nil { + panic(err) + } - if len(entriesArray) > 0 { - result := make([]har.Entry, 0) - for _, data := range entriesArray { - var pair tapApi.RequestResponsePair - if err := json.Unmarshal([]byte(data.Entry), &pair); err != nil { - continue - } - harEntry, err := utils.NewEntry(&pair) - if err != nil { - continue - } - if data.ResolvedSource != "" { - harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: data.ResolvedSource}) - } - if data.ResolvedDestination != "" { - harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: data.ResolvedDestination}) - harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, data.ResolvedDestination) - } + data := make(chan []byte) + meta := make(chan []byte) - // go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64 - if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil { - continue - } + defer func() { + data <- []byte(basenine.CloseChannel) + meta <- []byte(basenine.CloseChannel) + connection.Close() + }() - result = append(result, *harEntry) + lastTimeSynced := time.Time{} + + batch := make([]har.Entry, 0) + + handleDataChannel := func(wg *sync.WaitGroup, connection *basenine.Connection, data chan []byte) { + defer wg.Done() + for { + dataBytes := <-data + + if string(dataBytes) == basenine.CloseChannel { + return } - logger.Log.Infof("About to upload %v entries\n", len(result)) + var dataMap map[string]interface{} + err = json.Unmarshal(dataBytes, &dataMap) - body, jMarshalErr := json.Marshal(result) + var entry tapApi.MizuEntry + if err := json.Unmarshal([]byte(dataBytes), &entry); err != nil { + continue + } + harEntry, err := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime) + if err != nil { + continue + } + if entry.Source.Name != "" { + harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: entry.Source.Name}) + } + if entry.Destination.Name != "" { + harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entry.Destination.Name}) + harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entry.Destination.Name) + } + + // go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64 + if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil { + continue + } + + batch = append(batch, *harEntry) + + now := time.Now() + if lastTimeSynced.Add(time.Duration(uploadIntervalSec) * time.Second).After(now) { + continue + } + lastTimeSynced = now + + body, jMarshalErr := json.Marshal(batch) + batchSize := len(batch) if jMarshalErr != nil { analyzeInformation.Reset() logger.Log.Infof("Stopping sync entries") logger.Log.Fatal(jMarshalErr) } + batch = make([]har.Entry, 0) var in bytes.Buffer w := zlib.NewWriter(&in) @@ -273,18 +304,33 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva logger.Log.Info("Stopping sync entries") logger.Log.Fatal(postErr) } - analyzeInformation.SentCount += len(entriesArray) - logger.Log.Infof("Finish uploading %v entries to %s\n", len(entriesArray), GetTrafficDumpUrl(envPrefix, model)) + analyzeInformation.SentCount += batchSize - logger.Log.Infof("Uploaded %v entries until now", analyzeInformation.SentCount) - } else { - logger.Log.Infof("Nothing to upload") + if analyzeInformation.SentCount%SentCountLogInterval == 0 { + logger.Log.Infof("Uploaded %v entries until now", analyzeInformation.SentCount) + } } - - logger.Log.Infof("Sleeping for %v...\n", sleepTime) - time.Sleep(sleepTime) - timeFrom = timeTo } + + handleMetaChannel := func(wg *sync.WaitGroup, connection *basenine.Connection, meta chan []byte) { + defer wg.Done() + for { + metaBytes := <-meta + + if string(metaBytes) == basenine.CloseChannel { + return + } + } + } + + var wg sync.WaitGroup + go handleDataChannel(&wg, connection, data) + go handleMetaChannel(&wg, connection, meta) + wg.Add(2) + + connection.Query(query, data, meta) + + wg.Wait() } func UpdateAnalyzeStatus(callback func(data []byte)) { diff --git a/agent/pkg/utils/har.go b/agent/pkg/utils/har.go index a1a06be78..108e91b7b 100644 --- a/agent/pkg/utils/har.go +++ b/agent/pkg/utils/har.go @@ -10,7 +10,6 @@ import ( "github.com/google/martian/har" "github.com/up9inc/mizu/shared/logger" - "github.com/up9inc/mizu/tap/api" ) // Keep it because we might want cookies in the future @@ -120,13 +119,11 @@ func BuildPostParams(rawParams []interface{}) []har.Param { return params } -func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error) { - reqDetails := request.Payload.(map[string]interface{})["details"].(map[string]interface{}) +func NewRequest(request map[string]interface{}) (harRequest *har.Request, err error) { + headers, host, scheme, authority, path, _ := BuildHeaders(request["_headers"].([]interface{})) + cookies := make([]har.Cookie, 0) // BuildCookies(request["_cookies"].([]interface{})) - headers, host, scheme, authority, path, _ := BuildHeaders(reqDetails["headers"].([]interface{})) - cookies := make([]har.Cookie, 0) // BuildCookies(reqDetails["cookies"].([]interface{})) - - postData, _ := reqDetails["postData"].(map[string]interface{}) + postData, _ := request["postData"].(map[string]interface{}) mimeType, _ := postData["mimeType"] if mimeType == nil || len(mimeType.(string)) == 0 { mimeType = "text/html" @@ -138,7 +135,7 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error } queryString := make([]har.QueryString, 0) - for _, _qs := range reqDetails["queryString"].([]interface{}) { + for _, _qs := range request["_queryString"].([]interface{}) { qs := _qs.(map[string]interface{}) queryString = append(queryString, har.QueryString{ Name: qs["name"].(string), @@ -146,7 +143,7 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error }) } - url := fmt.Sprintf("http://%s%s", host, reqDetails["url"].(string)) + url := fmt.Sprintf("http://%s%s", host, request["url"].(string)) if strings.HasPrefix(mimeType.(string), "application/grpc") { url = fmt.Sprintf("%s://%s%s", scheme, authority, path) } @@ -157,9 +154,9 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error } harRequest = &har.Request{ - Method: reqDetails["method"].(string), + Method: request["method"].(string), URL: url, - HTTPVersion: reqDetails["httpVersion"].(string), + HTTPVersion: request["httpVersion"].(string), HeadersSize: -1, BodySize: int64(bytes.NewBufferString(postDataText).Len()), QueryString: queryString, @@ -175,13 +172,11 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error return } -func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err error) { - resDetails := response.Payload.(map[string]interface{})["details"].(map[string]interface{}) +func NewResponse(response map[string]interface{}) (harResponse *har.Response, err error) { + headers, _, _, _, _, _status := BuildHeaders(response["_headers"].([]interface{})) + cookies := make([]har.Cookie, 0) // BuildCookies(response["_cookies"].([]interface{})) - headers, _, _, _, _, _status := BuildHeaders(resDetails["headers"].([]interface{})) - cookies := make([]har.Cookie, 0) // BuildCookies(resDetails["cookies"].([]interface{})) - - content, _ := resDetails["content"].(map[string]interface{}) + content, _ := response["content"].(map[string]interface{}) mimeType, _ := content["mimeType"] if mimeType == nil || len(mimeType.(string)) == 0 { mimeType = "text/html" @@ -200,9 +195,11 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e Size: int64(len(bodyText)), } - status := int(resDetails["status"].(float64)) + status := int(response["status"].(float64)) if strings.HasPrefix(mimeType.(string), "application/grpc") { - status, err = strconv.Atoi(_status) + if _status != "" { + status, err = strconv.Atoi(_status) + } if err != nil { logger.Log.Errorf("Failed converting status to int %s (%v,%+v)", err, err, err) return nil, errors.New("failed converting response status to int for HAR") @@ -210,9 +207,9 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e } harResponse = &har.Response{ - HTTPVersion: resDetails["httpVersion"].(string), + HTTPVersion: response["httpVersion"].(string), Status: status, - StatusText: resDetails["statusText"].(string), + StatusText: response["statusText"].(string), HeadersSize: -1, BodySize: int64(bytes.NewBufferString(bodyText).Len()), Headers: headers, @@ -222,34 +219,33 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e return } -func NewEntry(pair *api.RequestResponsePair) (*har.Entry, error) { - harRequest, err := NewRequest(&pair.Request) +func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*har.Entry, error) { + harRequest, err := NewRequest(request) if err != nil { logger.Log.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err) return nil, errors.New("failed converting request to HAR") } - harResponse, err := NewResponse(&pair.Response) + harResponse, err := NewResponse(response) if err != nil { logger.Log.Errorf("Failed converting response to HAR %s (%v,%+v)", err, err, err) return nil, errors.New("failed converting response to HAR") } - totalTime := pair.Response.CaptureTime.Sub(pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds() - if totalTime < 1 { - totalTime = 1 + if elapsedTime < 1 { + elapsedTime = 1 } harEntry := har.Entry{ - StartedDateTime: pair.Request.CaptureTime, - Time: totalTime, + StartedDateTime: startTime, + Time: elapsedTime, Request: harRequest, Response: harResponse, Cache: &har.Cache{}, Timings: &har.Timings{ Send: -1, Wait: -1, - Receive: totalTime, + Receive: elapsedTime, }, } diff --git a/agent/pkg/utils/truncating_logger.go b/agent/pkg/utils/truncating_logger.go deleted file mode 100644 index 8302d5512..000000000 --- a/agent/pkg/utils/truncating_logger.go +++ /dev/null @@ -1,60 +0,0 @@ -package utils - -import ( - "context" - "fmt" - "time" - - loggerShared "github.com/up9inc/mizu/shared/logger" - "gorm.io/gorm/logger" - "gorm.io/gorm/utils" -) - -// TruncatingLogger implements the gorm logger.Interface interface. Its purpose is to act as gorm's logger while truncating logs to a max of 50 characters to minimise the performance impact -type TruncatingLogger struct { - LogLevel logger.LogLevel - SlowThreshold time.Duration -} - -func (truncatingLogger *TruncatingLogger) LogMode(logLevel logger.LogLevel) logger.Interface { - truncatingLogger.LogLevel = logLevel - return truncatingLogger -} - -func (truncatingLogger *TruncatingLogger) Info(_ context.Context, message string, __ ...interface{}) { - if truncatingLogger.LogLevel < logger.Info { - return - } - loggerShared.Log.Errorf("gorm info: %.150s", message) -} - -func (truncatingLogger *TruncatingLogger) Warn(_ context.Context, message string, __ ...interface{}) { - if truncatingLogger.LogLevel < logger.Warn { - return - } - loggerShared.Log.Errorf("gorm warning: %.150s", message) -} - -func (truncatingLogger *TruncatingLogger) Error(_ context.Context, message string, __ ...interface{}) { - if truncatingLogger.LogLevel < logger.Error { - return - } - loggerShared.Log.Errorf("gorm error: %.150s", message) -} - -func (truncatingLogger *TruncatingLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) { - if truncatingLogger.LogLevel == logger.Silent { - return - } - elapsed := time.Since(begin) - if err != nil { - sql, rows := fc() // copied into every condition as this is a potentially heavy operation best done only when necessary - truncatingLogger.Error(ctx, fmt.Sprintf("Error in %s: %v - elapsed: %fs affected rows: %d, sql: %s", utils.FileWithLineNum(), err, elapsed.Seconds(), rows, sql)) - } else if truncatingLogger.LogLevel >= logger.Warn && elapsed > truncatingLogger.SlowThreshold { - sql, rows := fc() - truncatingLogger.Warn(ctx, fmt.Sprintf("Slow sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql)) - } else if truncatingLogger.LogLevel >= logger.Info { - sql, rows := fc() - truncatingLogger.Info(ctx, fmt.Sprintf("Sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql)) - } -} diff --git a/agent/pkg/utils/utils.go b/agent/pkg/utils/utils.go index 4838fe666..95b92e288 100644 --- a/agent/pkg/utils/utils.go +++ b/agent/pkg/utils/utils.go @@ -3,11 +3,12 @@ package utils import ( "context" "fmt" + "mizuserver/pkg/providers" "net/http" "net/url" "os" "os/signal" - "reflect" + "strings" "syscall" "time" @@ -44,15 +45,13 @@ func StartServer(app *gin.Engine) { } } -func ReverseSlice(data interface{}) { - value := reflect.ValueOf(data) - valueLen := value.Len() - for i := 0; i <= int((valueLen-1)/2); i++ { - reverseIndex := valueLen - 1 - i - tmp := value.Index(reverseIndex).Interface() - value.Index(reverseIndex).Set(value.Index(i)) - value.Index(i).Set(reflect.ValueOf(tmp)) +func GetTappedPodsStatus() []shared.TappedPodStatus { + tappedPodsStatus := make([]shared.TappedPodStatus, 0) + for _, pod := range providers.TapStatus.Pods { + isTapped := strings.ToLower(providers.TappersStatus[pod.NodeName].Status) == "started" + tappedPodsStatus = append(tappedPodsStatus, shared.TappedPodStatus{Name: pod.Name, Namespace: pod.Namespace, IsTapped: isTapped}) } + return tappedPodsStatus } func CheckErr(e error) { diff --git a/assets/mizu-ui.png b/assets/mizu-ui.png index 29a1d17ba..0a27eface 100644 Binary files a/assets/mizu-ui.png and b/assets/mizu-ui.png differ diff --git a/assets/validation-example1.png b/assets/validation-example1.png index cac14b6d2..0e6c6ee89 100644 Binary files a/assets/validation-example1.png and b/assets/validation-example1.png differ diff --git a/assets/validation-example2.png b/assets/validation-example2.png index d870c39e2..5bc07683d 100644 Binary files a/assets/validation-example2.png and b/assets/validation-example2.png differ diff --git a/cli/Makefile b/cli/Makefile index 48f42b94e..8275a12da 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -18,8 +18,9 @@ build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables go build -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \ -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \ -X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \ + -X 'github.com/up9inc/mizu/cli/mizu.Platform=$(SUFFIX)' \ -X 'github.com/up9inc/mizu/cli/mizu.SemVer=$(SEM_VER)'" \ - -o bin/mizu_$(SUFFIX) mizu.go + -o bin/mizu_$(SUFFIX) mizu.go (cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256) build-all: ## Build for all supported platforms. diff --git a/cli/apiserver/provider.go b/cli/apiserver/provider.go index b032cd4c8..dc5dff5e9 100644 --- a/cli/apiserver/provider.go +++ b/cli/apiserver/provider.go @@ -3,11 +3,12 @@ package apiserver import ( "bytes" "encoding/json" - "errors" "fmt" + "io" "io/ioutil" "net/http" "net/url" + "strings" "time" "github.com/up9inc/mizu/shared/kubernetes" @@ -41,7 +42,7 @@ func (provider *Provider) TestConnection() error { retriesLeft := provider.retries for retriesLeft > 0 { if _, err := provider.GetHealthStatus(); err != nil { - logger.Log.Debugf("[ERROR] api server not ready yet %v", err) + logger.Log.Debugf("api server not ready yet %v", err) } else { logger.Log.Debugf("connection test to api server passed successfully") break @@ -61,7 +62,14 @@ func (provider *Provider) GetHealthStatus() (*shared.HealthResponse, error) { if response, err := provider.client.Get(healthUrl); err != nil { return nil, err } else if response.StatusCode > 299 { - return nil, errors.New(fmt.Sprintf("status code: %d", response.StatusCode)) + responseBody := new(strings.Builder) + + if _, err := io.Copy(responseBody, response.Body); err != nil { + return nil, fmt.Errorf("status code: %d - (bad response - %v)", response.StatusCode, err) + } else { + singleLineResponse := strings.ReplaceAll(responseBody.String(), "\n", "") + return nil, fmt.Errorf("status code: %d - (response - %v)", response.StatusCode, singleLineResponse) + } } else { defer response.Body.Close() @@ -73,6 +81,23 @@ func (provider *Provider) GetHealthStatus() (*shared.HealthResponse, error) { } } +func (provider *Provider) ReportTapperStatus(tapperStatus shared.TapperStatus) error { + tapperStatusUrl := fmt.Sprintf("%s/status/tapperStatus", provider.url) + + if jsonValue, err := json.Marshal(tapperStatus); err != nil { + return fmt.Errorf("failed Marshal the tapper status %w", err) + } else { + if response, err := provider.client.Post(tapperStatusUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil { + return fmt.Errorf("failed sending to API server the tapped pods %w", err) + } else if response.StatusCode != 200 { + return fmt.Errorf("failed sending to API server the tapper status, response status code %v", response.StatusCode) + } else { + logger.Log.Debugf("Reported to server API about tapper status: %v", tapperStatus) + return nil + } + } +} + func (provider *Provider) ReportTappedPods(pods []core.Pod) error { tappedPodsUrl := fmt.Sprintf("%s/status/tappedPods", provider.url) diff --git a/cli/cmd/common.go b/cli/cmd/common.go index bb6e0a992..822335c8e 100644 --- a/cli/cmd/common.go +++ b/cli/cmd/common.go @@ -4,9 +4,15 @@ import ( "context" "errors" "fmt" + "github.com/up9inc/mizu/cli/apiserver" + "github.com/up9inc/mizu/cli/mizu" + "github.com/up9inc/mizu/cli/mizu/fsUtils" + "github.com/up9inc/mizu/cli/telemetry" "os" "os/signal" + "path" "syscall" + "time" "github.com/up9inc/mizu/cli/config" "github.com/up9inc/mizu/cli/config/configStructs" @@ -64,3 +70,42 @@ func handleKubernetesProviderError(err error) { logger.Log.Error(err) } } + +func finishMizuExecution(kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider) { + telemetry.ReportAPICalls(apiProvider) + removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) + defer cancel() + dumpLogsIfNeeded(removalCtx, kubernetesProvider) + cleanUpMizuResources(removalCtx, cancel, kubernetesProvider) +} + +func dumpLogsIfNeeded(ctx context.Context, kubernetesProvider *kubernetes.Provider) { + if !config.Config.DumpLogs { + return + } + mizuDir := mizu.GetMizuFolderPath() + filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05"))) + if err := fsUtils.DumpLogs(ctx, kubernetesProvider, filePath); err != nil { + logger.Log.Errorf("Failed dump logs %v", err) + } +} + +func cleanUpMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) { + logger.Log.Infof("\nRemoving mizu resources") + + var leftoverResources []string + + if config.Config.IsNsRestrictedMode() { + leftoverResources = cleanUpRestrictedMode(ctx, kubernetesProvider) + } else { + leftoverResources = cleanUpNonRestrictedMode(ctx, cancel, kubernetesProvider) + } + + if len(leftoverResources) > 0 { + errMsg := fmt.Sprintf("Failed to remove the following resources, for more info check logs at %s:", fsUtils.GetLogFilePath()) + for _, resource := range leftoverResources { + errMsg += "\n- " + resource + } + logger.Log.Errorf(uiUtils.Error, errMsg) + } +} diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 9c40d6647..97168ce31 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -23,6 +23,7 @@ Further info is available at https://github.com/up9inc/mizu`, if err := config.InitConfig(cmd); err != nil { logger.Log.Fatal(err) } + return nil }, } diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index f4ed44605..f93a8f65a 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -3,6 +3,7 @@ package cmd import ( "errors" "fmt" + "github.com/up9inc/mizu/cli/up9" "os" "github.com/creasty/defaults" @@ -62,6 +63,12 @@ Supported protocols are HTTP and gRPC.`, logger.Log.Errorf("failed to log in, err: %v", err) return nil } + } else if isValidToken := up9.IsTokenValid(config.Config.Auth.Token, config.Config.Auth.EnvName); !isValidToken { + logger.Log.Errorf("Token is not valid, please log in again to continue") + if err := auth.Login(); err != nil { + logger.Log.Errorf("failed to log in, err: %v", err) + return nil + } } } } @@ -113,4 +120,5 @@ func init() { tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file path with policy rules") tapCmd.Flags().String(configStructs.ContractFile, defaultTapConfig.ContractFile, "OAS/Swagger file to validate to monitor the contracts") tapCmd.Flags().Bool(configStructs.DaemonModeTapName, defaultTapConfig.DaemonMode, "Run mizu in daemon mode, detached from the cli") + tapCmd.Flags().Bool(configStructs.IstioName, defaultTapConfig.Istio, "Record decrypted traffic if the cluster configured with istio and mtls") } diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index f0238d733..90b7eb071 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -4,27 +4,27 @@ import ( "context" "errors" "fmt" - "github.com/up9inc/mizu/cli/cmd/goUtils" "io/ioutil" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "path" "regexp" "strings" "time" + "k8s.io/apimachinery/pkg/util/intstr" + "github.com/getkin/kin-openapi/openapi3" + "gopkg.in/yaml.v3" + core "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "github.com/up9inc/mizu/cli/apiserver" + "github.com/up9inc/mizu/cli/cmd/goUtils" "github.com/up9inc/mizu/cli/config" "github.com/up9inc/mizu/cli/config/configStructs" "github.com/up9inc/mizu/cli/errormessage" - "gopkg.in/yaml.v3" - core "k8s.io/api/core/v1" - "github.com/up9inc/mizu/cli/mizu" "github.com/up9inc/mizu/cli/mizu/fsUtils" - "github.com/up9inc/mizu/cli/telemetry" "github.com/up9inc/mizu/cli/uiUtils" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/kubernetes" @@ -35,6 +35,9 @@ import ( const cleanupTimeout = time.Minute type tapState struct { + startTime time.Time + targetNamespaces []string + apiServerService *core.Service tapperSyncer *kubernetes.MizuTapperSyncer mizuServiceAccountExists bool @@ -44,6 +47,8 @@ var state tapState var apiProvider *apiserver.Provider func RunMizuTap() { + state.startTime = time.Now() + mizuApiFilteringOptions, err := getMizuApiFilteringOptions() apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout) if err != nil { @@ -92,16 +97,16 @@ func RunMizuTap() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() // cancel will be called when this function exits - targetNamespaces := getNamespaces(kubernetesProvider) + state.targetNamespaces = getNamespaces(kubernetesProvider) - serializedMizuConfig, err := config.GetSerializedMizuAgentConfig(targetNamespaces, mizuApiFilteringOptions) + serializedMizuConfig, err := config.GetSerializedMizuAgentConfig(state.targetNamespaces, mizuApiFilteringOptions) if err != nil { logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error composing mizu config: %v", errormessage.FormatError(err))) return } if config.Config.IsNsRestrictedMode() { - if len(targetNamespaces) != 1 || !shared.Contains(targetNamespaces, config.Config.MizuResourcesNamespace) { + if len(state.targetNamespaces) != 1 || !shared.Contains(state.targetNamespaces, config.Config.MizuResourcesNamespace) { logger.Log.Errorf("Not supported mode. Mizu can't resolve IPs in other namespaces when running in namespace restricted mode.\n"+ "You can use the same namespace for --%s and --%s", configStructs.NamespacesTapName, config.MizuResourcesNamespaceConfigName) return @@ -109,18 +114,23 @@ func RunMizuTap() { } var namespacesStr string - if !shared.Contains(targetNamespaces, kubernetes.K8sAllNamespaces) { - namespacesStr = fmt.Sprintf("namespaces \"%s\"", strings.Join(targetNamespaces, "\", \"")) + if !shared.Contains(state.targetNamespaces, kubernetes.K8sAllNamespaces) { + namespacesStr = fmt.Sprintf("namespaces \"%s\"", strings.Join(state.targetNamespaces, "\", \"")) } else { namespacesStr = "all namespaces" } logger.Log.Infof("Tapping pods in %s", namespacesStr) + if err := printTappedPodsPreview(ctx, kubernetesProvider, state.targetNamespaces); err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error listing pods: %v", errormessage.FormatError(err))) + } + if config.Config.Tap.DryRun { return } + logger.Log.Infof("Waiting for Mizu Agent to start...") if err := createMizuResources(ctx, cancel, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig); err != nil { logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err))) @@ -133,7 +143,7 @@ func RunMizuTap() { return } if config.Config.Tap.DaemonMode { - if err := handleDaemonModePostCreation(cancel, kubernetesProvider); err != nil { + if err := handleDaemonModePostCreation(ctx, cancel, kubernetesProvider, state.targetNamespaces); err != nil { defer finishMizuExecution(kubernetesProvider, apiProvider) cancel() } else { @@ -142,45 +152,44 @@ func RunMizuTap() { } else { defer finishMizuExecution(kubernetesProvider, apiProvider) - if err = startTapperSyncer(ctx, cancel, kubernetesProvider, targetNamespaces, *mizuApiFilteringOptions); err != nil { - logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error starting mizu tapper syncer: %v", err)) - cancel() - } - + go goUtils.HandleExcWrapper(watchApiServerEvents, ctx, kubernetesProvider, cancel) go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel) - go goUtils.HandleExcWrapper(watchTapperPod, ctx, kubernetesProvider, cancel) // block until exit signal or error waitForFinish(ctx, cancel) } } -func handleDaemonModePostCreation(cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) error { +func handleDaemonModePostCreation(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, namespaces []string) error { apiProvider := apiserver.NewProvider(GetApiServerUrl(), 90, 1*time.Second) if err := waitForDaemonModeToBeReady(cancel, kubernetesProvider, apiProvider); err != nil { return err } - if err := printDaemonModeTappedPods(apiProvider); err != nil { - return err - } return nil } -func printDaemonModeTappedPods(apiProvider *apiserver.Provider) error { - if healthStatus, err := apiProvider.GetHealthStatus(); err != nil { +/* +this function is a bit problematic as it might be detached from the actual pods the mizu api server will tap. +The alternative would be to wait for api server to be ready and then query it for the pods it listens to, this has +the arguably worse drawback of taking a relatively very long time before the user sees which pods are targeted, if any. +*/ +func printTappedPodsPreview(ctx context.Context, kubernetesProvider *kubernetes.Provider, namespaces []string) error { + if matchingPods, err := kubernetesProvider.ListAllRunningPodsMatchingRegex(ctx, config.Config.Tap.PodRegex(), namespaces); err != nil { return err } else { - for _, tappedPod := range healthStatus.TapStatus.Pods { + if len(matchingPods) == 0 { + printNoPodsFoundSuggestion(namespaces) + } + for _, tappedPod := range matchingPods { logger.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", tappedPod.Name)) } + return nil } - return nil } func waitForDaemonModeToBeReady(cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider) error { - logger.Log.Info("Waiting for mizu to be ready... (may take a few minutes)") go startProxyReportErrorIfAny(kubernetesProvider, cancel) // TODO: TRA-3903 add a smarter test to see that tapping/pod watching is functioning properly @@ -191,7 +200,7 @@ func waitForDaemonModeToBeReady(cancel context.CancelFunc, kubernetesProvider *k return nil } -func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, mizuApiFilteringOptions api.TrafficFilteringOptions) error { +func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, mizuApiFilteringOptions api.TrafficFilteringOptions, startTime time.Time) error { tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{ TargetNamespaces: targetNamespaces, PodFilterRegex: *config.Config.Tap.PodRegex(), @@ -199,28 +208,17 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider AgentImage: config.Config.AgentImage, TapperResources: config.Config.Tap.TapperResources, ImagePullPolicy: config.Config.ImagePullPolicy(), - DumpLogs: config.Config.DumpLogs, + LogLevel: config.Config.LogLevel(), IgnoredUserAgents: config.Config.Tap.IgnoredUserAgents, MizuApiFilteringOptions: mizuApiFilteringOptions, MizuServiceAccountExists: state.mizuServiceAccountExists, - }) + Istio: config.Config.Tap.Istio, + }, startTime) if err != nil { return err } - for _, tappedPod := range tapperSyncer.CurrentlyTappedPods { - logger.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", tappedPod.Name)) - } - - if len(tapperSyncer.CurrentlyTappedPods) == 0 { - var suggestionStr string - if !shared.Contains(targetNamespaces, kubernetes.K8sAllNamespaces) { - suggestionStr = ". Select a different namespace with -n or tap all namespaces with -A" - } - logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any pods matching the regex argument%s", suggestionStr)) - } - go func() { for { select { @@ -239,6 +237,14 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider if err := apiProvider.ReportTappedPods(tapperSyncer.CurrentlyTappedPods); err != nil { logger.Log.Debugf("[Error] failed update tapped pods %v", err) } + case tapperStatus, ok := <-tapperSyncer.TapperStatusChangedOut: + if !ok { + logger.Log.Debug("mizuTapperSyncer tapper status changed channel closed, ending listener loop") + return + } + if err := apiProvider.ReportTapperStatus(tapperStatus); err != nil { + logger.Log.Debugf("[Error] failed update tapper status %v", err) + } case <-ctx.Done(): logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done") return @@ -251,6 +257,14 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider return nil } +func printNoPodsFoundSuggestion(targetNamespaces []string) { + var suggestionStr string + if !shared.Contains(targetNamespaces, kubernetes.K8sAllNamespaces) { + suggestionStr = ". You can also try selecting a different namespace with -n or tap all namespaces with -A" + } + logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any currently running pods that match the regex argument, mizu will automatically tap matching pods if any are created later%s", suggestionStr)) +} + func getErrorDisplayTextForK8sTapManagerError(err kubernetes.K8sTapManagerError) string { switch err.TapManagerReason { case kubernetes.TapManagerPodListError: @@ -281,7 +295,7 @@ func createMizuResources(ctx context.Context, cancel context.CancelFunc, kuberne } if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig); err != nil { - logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err))) + logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v", errormessage.FormatError(err))) } var err error @@ -309,6 +323,7 @@ func createMizuResources(ctx context.Context, cancel context.CancelFunc, kuberne MaxEntriesDBSizeBytes: config.Config.Tap.MaxEntriesDBSizeBytes(), Resources: config.Config.Tap.ApiServerResources, ImagePullPolicy: config.Config.ImagePullPolicy(), + LogLevel: config.Config.LogLevel(), } if config.Config.Tap.DaemonMode { @@ -357,27 +372,25 @@ func createMizuApiServerPod(ctx context.Context, kubernetesProvider *kubernetes. } func createMizuApiServerDeployment(ctx context.Context, kubernetesProvider *kubernetes.Provider, opts *kubernetes.ApiServerOptions) error { - isDefaultStorageClassAvailable, err := kubernetesProvider.IsDefaultStorageProviderAvailable(ctx) volumeClaimCreated := false - if err != nil { - return err - } - if isDefaultStorageClassAvailable { - if _, err = kubernetesProvider.CreatePersistentVolumeClaim(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName, config.Config.Tap.MaxEntriesDBSizeBytes()+mizu.DaemonModePersistentVolumeSizeBufferBytes); err != nil { - logger.Log.Warningf(uiUtils.Yellow, "An error has occured while creating a persistent volume claim for mizu, this will mean that mizu's data will be lost on pod restart") - logger.Log.Debugf("error creating persistent volume claim: %v", err) - } else { - volumeClaimCreated = true - } - } else { - logger.Log.Warningf(uiUtils.Yellow, "Could not find default volume provider in this cluster, this will mean that mizu's data will be lost on pod restart") + if !config.Config.Tap.NoPersistentVolumeClaim { + volumeClaimCreated = TryToCreatePersistentVolumeClaim(ctx, kubernetesProvider) } pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, volumeClaimCreated, kubernetes.PersistentVolumeClaimName) if err != nil { return err } - + pod.Spec.Containers[0].LivenessProbe = &core.Probe{ + Handler: core.Handler{ + HTTPGet: &core.HTTPGetAction{ + Path: "/echo", + Port: intstr.FromInt(shared.DefaultApiServerPort), + }, + }, + InitialDelaySeconds: 1, + PeriodSeconds: 10, + } if _, err = kubernetesProvider.CreateDeployment(ctx, config.Config.MizuResourcesNamespace, opts.PodName, pod); err != nil { return err } @@ -385,6 +398,26 @@ func createMizuApiServerDeployment(ctx context.Context, kubernetesProvider *kube return nil } +func TryToCreatePersistentVolumeClaim(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool { + isDefaultStorageClassAvailable, err := kubernetesProvider.IsDefaultStorageProviderAvailable(ctx) + if err != nil { + logger.Log.Warningf(uiUtils.Yellow, "An error occured when checking if a default storage provider exists in this cluster, this means mizu data will be lost on mizu-api-server pod restart") + logger.Log.Debugf("error checking if default storage class exists: %v", err) + return false + } else if !isDefaultStorageClassAvailable { + logger.Log.Warningf(uiUtils.Yellow, "Could not find default storage provider in this cluster, this means mizu data will be lost on mizu-api-server pod restart") + return false + } + + if _, err = kubernetesProvider.CreatePersistentVolumeClaim(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName, config.Config.Tap.MaxEntriesDBSizeBytes()+mizu.DaemonModePersistentVolumeSizeBufferBytes); err != nil { + logger.Log.Warningf(uiUtils.Yellow, "An error has occured while creating a persistent volume claim for mizu, this means mizu data will be lost on mizu-api-server pod restart") + logger.Log.Debugf("error creating persistent volume claim: %v", err) + return false + } + + return true +} + func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) { var compiledRegexSlice []*api.SerializableRegexp @@ -419,45 +452,6 @@ func getSyncEntriesConfig() *shared.SyncEntriesConfig { } } -func finishMizuExecution(kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider) { - telemetry.ReportAPICalls(apiProvider) - removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) - defer cancel() - dumpLogsIfNeeded(removalCtx, kubernetesProvider) - cleanUpMizuResources(removalCtx, cancel, kubernetesProvider) -} - -func dumpLogsIfNeeded(ctx context.Context, kubernetesProvider *kubernetes.Provider) { - if !config.Config.DumpLogs { - return - } - mizuDir := mizu.GetMizuFolderPath() - filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05"))) - if err := fsUtils.DumpLogs(ctx, kubernetesProvider, filePath); err != nil { - logger.Log.Errorf("Failed dump logs %v", err) - } -} - -func cleanUpMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) { - logger.Log.Infof("\nRemoving mizu resources\n") - - var leftoverResources []string - - if config.Config.IsNsRestrictedMode() { - leftoverResources = cleanUpRestrictedMode(ctx, kubernetesProvider) - } else { - leftoverResources = cleanUpNonRestrictedMode(ctx, cancel, kubernetesProvider) - } - - if len(leftoverResources) > 0 { - errMsg := fmt.Sprintf("Failed to remove the following resources, for more info check logs at %s:", fsUtils.GetLogFilePath()) - for _, resource := range leftoverResources { - errMsg += "\n- " + resource - } - logger.Log.Errorf(uiUtils.Error, errMsg) - } -} - func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.Provider) []string { leftoverResources := make([]string, 0) @@ -568,67 +562,43 @@ func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, k func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) { podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", kubernetes.ApiServerPodName)) - added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex) + podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex) + eventChan, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper) isPodReady := false timeAfter := time.After(25 * time.Second) for { select { - case _, ok := <-added: + case wEvent, ok := <-eventChan: if !ok { - added = nil + eventChan = nil continue } - logger.Log.Debugf("Watching API Server pod loop, added") - case _, ok := <-removed: - if !ok { - removed = nil - continue - } - - logger.Log.Infof("%s removed", kubernetes.ApiServerPodName) - cancel() - return - case modifiedPod, ok := <-modified: - if !ok { - modified = nil - continue - } - - logger.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase) - - if modifiedPod.Status.Phase == core.PodPending { - if modifiedPod.Status.Conditions[0].Type == core.PodScheduled && modifiedPod.Status.Conditions[0].Status != core.ConditionTrue { - logger.Log.Debugf("Wasn't able to deploy the API server. Reason: \"%s\"", modifiedPod.Status.Conditions[0].Message) - logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Wasn't able to deploy the API server, for more info check logs at %s", fsUtils.GetLogFilePath())) + switch wEvent.Type { + case kubernetes.EventAdded: + logger.Log.Debugf("Watching API Server pod loop, added") + case kubernetes.EventDeleted: + logger.Log.Infof("%s removed", kubernetes.ApiServerPodName) + cancel() + return + case kubernetes.EventModified: + modifiedPod, err := wEvent.ToPod() + if err != nil { + logger.Log.Errorf(uiUtils.Error, err) cancel() - break + continue } - if len(modifiedPod.Status.ContainerStatuses) > 0 && modifiedPod.Status.ContainerStatuses[0].State.Waiting != nil && modifiedPod.Status.ContainerStatuses[0].State.Waiting.Reason == "ErrImagePull" { - logger.Log.Debugf("Wasn't able to deploy the API server. (ErrImagePull) Reason: \"%s\"", modifiedPod.Status.ContainerStatuses[0].State.Waiting.Message) - logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Wasn't able to deploy the API server: failed to pull the image, for more info check logs at %v", fsUtils.GetLogFilePath())) - cancel() - break - } - } + logger.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase) - if modifiedPod.Status.Phase == core.PodRunning && !isPodReady { - isPodReady = true - go startProxyReportErrorIfAny(kubernetesProvider, cancel) - - url := GetApiServerUrl() - if err := apiProvider.TestConnection(); err != nil { - logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath())) - cancel() - break - } - - logger.Log.Infof("Mizu is available at %s\n", url) - uiUtils.OpenBrowser(url) - if err := apiProvider.ReportTappedPods(state.tapperSyncer.CurrentlyTappedPods); err != nil { - logger.Log.Debugf("[Error] failed update tapped pods %v", err) + if modifiedPod.Status.Phase == core.PodRunning && !isPodReady { + isPodReady = true + postApiServerStarted(ctx, kubernetesProvider, cancel, err) } + case kubernetes.EventBookmark: + break + case kubernetes.EventError: + break } case err, ok := <-errorChan: if !ok { @@ -651,72 +621,77 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi } } -func watchTapperPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) { - podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", kubernetes.TapperDaemonSetName)) - added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex) - var prevPodPhase core.PodPhase +func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) { + podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s", kubernetes.ApiServerPodName)) + eventWatchHelper := kubernetes.NewEventWatchHelper(kubernetesProvider, podExactRegex, "pod") + eventChan, errorChan := kubernetes.FilteredWatch(ctx, eventWatchHelper, []string{config.Config.MizuResourcesNamespace}, eventWatchHelper) for { select { - case addedPod, ok := <-added: + case wEvent, ok := <-eventChan: if !ok { - added = nil + eventChan = nil continue } - logger.Log.Debugf("Tapper is created [%s]", addedPod.Name) - case removedPod, ok := <-removed: - if !ok { - removed = nil + event, err := wEvent.ToEvent() + if err != nil { + logger.Log.Errorf(fmt.Sprintf("Error parsing Mizu resource event: %+v", err)) + } + + if state.startTime.After(event.CreationTimestamp.Time) { continue } - logger.Log.Debugf("Tapper is removed [%s]", removedPod.Name) - case modifiedPod, ok := <-modified: - if !ok { - modified = nil - continue - } + logger.Log.Debugf( + fmt.Sprintf("Watching API server events loop, event %s, time: %v, resource: %s (%s), reason: %s, note: %s", + event.Name, + event.CreationTimestamp.Time, + event.Regarding.Name, + event.Regarding.Kind, + event.Reason, + event.Note)) - if modifiedPod.Status.Phase == core.PodPending && modifiedPod.Status.Conditions[0].Type == core.PodScheduled && modifiedPod.Status.Conditions[0].Status != core.ConditionTrue { - logger.Log.Infof(uiUtils.Red, fmt.Sprintf("Wasn't able to deploy the tapper %s. Reason: \"%s\"", modifiedPod.Name, modifiedPod.Status.Conditions[0].Message)) + switch event.Reason { + case "FailedScheduling", "Failed": + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Mizu API Server status: %s - %s", event.Reason, event.Note)) cancel() break } - - podStatus := modifiedPod.Status - if podStatus.Phase == core.PodPending && prevPodPhase == podStatus.Phase { - logger.Log.Debugf("Tapper %s is %s", modifiedPod.Name, strings.ToLower(string(podStatus.Phase))) - continue - } - prevPodPhase = podStatus.Phase - - if podStatus.Phase == core.PodRunning { - state := podStatus.ContainerStatuses[0].State - if state.Terminated != nil { - switch state.Terminated.Reason { - case "OOMKilled": - logger.Log.Infof(uiUtils.Red, fmt.Sprintf("Tapper %s was terminated (reason: OOMKilled). You should consider increasing machine resources.", modifiedPod.Name)) - } - } - } - - logger.Log.Debugf("Tapper %s is %s", modifiedPod.Name, strings.ToLower(string(podStatus.Phase))) case err, ok := <-errorChan: if !ok { errorChan = nil continue } - logger.Log.Errorf("[Error] Error in mizu tapper watch, err: %v", err) - cancel() - + logger.Log.Errorf("Watching API server events loop, error: %+v", err) case <-ctx.Done(): - logger.Log.Debugf("Watching tapper pod loop, ctx done") + logger.Log.Debugf("Watching API server events loop, ctx done") return } } } +func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, err error) { + go startProxyReportErrorIfAny(kubernetesProvider, cancel) + + url := GetApiServerUrl() + if err := apiProvider.TestConnection(); err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath())) + cancel() + return + } + options, _ := getMizuApiFilteringOptions() + if err = startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error starting mizu tapper syncer: %v", err)) + cancel() + } + + logger.Log.Infof("Mizu is available at %s", url) + if !config.Config.HeadlessMode { + uiUtils.OpenBrowser(url) + } +} + func getNamespaces(kubernetesProvider *kubernetes.Provider) []string { if config.Config.Tap.AllNamespaces { return []string{kubernetes.K8sAllNamespaces} diff --git a/cli/cmd/viewRunner.go b/cli/cmd/viewRunner.go index a342b484c..c9c056be1 100644 --- a/cli/cmd/viewRunner.go +++ b/cli/cmd/viewRunner.go @@ -56,9 +56,11 @@ func runMizuView() { return } - logger.Log.Infof("Mizu is available at %s\n", url) + logger.Log.Infof("Mizu is available at %s", url) - uiUtils.OpenBrowser(url) + if !config.Config.HeadlessMode { + uiUtils.OpenBrowser(url) + } if isCompatible, err := version.CheckVersionCompatibility(apiServerProvider); err != nil { logger.Log.Errorf("Failed to check versions compatibility %v", err) diff --git a/cli/config/config.go b/cli/config/config.go index 386f4816a..dc9501fab 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -3,14 +3,15 @@ package config import ( "errors" "fmt" - "github.com/up9inc/mizu/tap/api" "io/ioutil" - "k8s.io/apimachinery/pkg/util/json" "os" "reflect" "strconv" "strings" + "github.com/up9inc/mizu/tap/api" + "k8s.io/apimachinery/pkg/util/json" + "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" @@ -51,6 +52,10 @@ func InitConfig(cmd *cobra.Command) error { cmd.Flags().Visit(initFlag) + if err := Config.validate(); err != nil { + return fmt.Errorf("config validation failed, err: %v", err) + } + finalConfigPrettified, _ := uiUtils.PrettyJson(Config) logger.Log.Debugf("Init config finished\n Final config: %v", finalConfigPrettified) @@ -391,12 +396,13 @@ func getMizuAgentConfig(targetNamespaces []string, mizuApiFilteringOptions *api. TargetNamespaces: targetNamespaces, AgentImage: Config.AgentImage, PullPolicy: Config.ImagePullPolicyStr, - DumpLogs: Config.DumpLogs, + LogLevel: Config.LogLevel(), IgnoredUserAgents: Config.Tap.IgnoredUserAgents, TapperResources: Config.Tap.TapperResources, MizuResourcesNamespace: Config.MizuResourcesNamespace, MizuApiFilteringOptions: *mizuApiFilteringOptions, - AgentDatabasePath: fmt.Sprintf("%s%s", shared.DataDirPath, "entries.db"), + AgentDatabasePath: shared.DataDirPath, + Istio: Config.Tap.Istio, } return &config, nil } diff --git a/cli/config/configStruct.go b/cli/config/configStruct.go index 171ed11d0..f0402bf2d 100644 --- a/cli/config/configStruct.go +++ b/cli/config/configStruct.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "github.com/op/go-logging" "github.com/up9inc/mizu/cli/config/configStructs" "github.com/up9inc/mizu/cli/mizu" v1 "k8s.io/api/core/v1" @@ -31,6 +32,16 @@ type ConfigStruct struct { DumpLogs bool `yaml:"dump-logs" default:"false"` KubeConfigPathStr string `yaml:"kube-config-path"` ConfigFilePath string `yaml:"config-path,omitempty" readonly:""` + HeadlessMode bool `yaml:"headless" default:"false"` + LogLevelStr string `yaml:"log-level,omitempty" default:"INFO" readonly:""` +} + +func(config *ConfigStruct) validate() error { + if _, err := logging.LogLevel(config.LogLevelStr); err != nil { + return fmt.Errorf("%s is not a valid log level, err: %v", config.LogLevelStr, err) + } + + return nil } func (config *ConfigStruct) SetDefaults() { @@ -59,3 +70,8 @@ func (config *ConfigStruct) KubeConfigPath() string { home := homedir.HomeDir() return filepath.Join(home, ".kube", "config") } + +func (config *ConfigStruct) LogLevel() logging.Level { + logLevel, _ := logging.LogLevel(config.LogLevelStr) + return logLevel +} diff --git a/cli/config/configStructs/tapConfig.go b/cli/config/configStructs/tapConfig.go index 38fb187db..43c29523d 100644 --- a/cli/config/configStructs/tapConfig.go +++ b/cli/config/configStructs/tapConfig.go @@ -3,9 +3,12 @@ package configStructs import ( "errors" "fmt" - "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/cli/uiUtils" + "github.com/up9inc/mizu/shared/logger" "regexp" + "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/shared/units" ) @@ -22,28 +25,31 @@ const ( EnforcePolicyFile = "traffic-validation-file" ContractFile = "contract" DaemonModeTapName = "daemon" + IstioName = "istio" ) type TapConfig struct { - UploadIntervalSec int `yaml:"upload-interval" default:"10"` - PodRegexStr string `yaml:"regex" default:".*"` - GuiPort uint16 `yaml:"gui-port" default:"8899"` - ProxyHost string `yaml:"proxy-host" default:"127.0.0.1"` - Namespaces []string `yaml:"namespaces"` - Analysis bool `yaml:"analysis" default:"false"` - AllNamespaces bool `yaml:"all-namespaces" default:"false"` - PlainTextFilterRegexes []string `yaml:"regex-masking"` - IgnoredUserAgents []string `yaml:"ignored-user-agents"` - DisableRedaction bool `yaml:"no-redact" default:"false"` - HumanMaxEntriesDBSize string `yaml:"max-entries-db-size" default:"200MB"` - DryRun bool `yaml:"dry-run" default:"false"` - Workspace string `yaml:"workspace"` - EnforcePolicyFile string `yaml:"traffic-validation-file"` - ContractFile string `yaml:"contract"` - AskUploadConfirmation bool `yaml:"ask-upload-confirmation" default:"true"` - ApiServerResources shared.Resources `yaml:"api-server-resources"` - TapperResources shared.Resources `yaml:"tapper-resources"` - DaemonMode bool `yaml:"daemon" default:"false"` + UploadIntervalSec int `yaml:"upload-interval" default:"10"` + PodRegexStr string `yaml:"regex" default:".*"` + GuiPort uint16 `yaml:"gui-port" default:"8899"` + ProxyHost string `yaml:"proxy-host" default:"127.0.0.1"` + Namespaces []string `yaml:"namespaces"` + Analysis bool `yaml:"analysis" default:"false"` + AllNamespaces bool `yaml:"all-namespaces" default:"false"` + PlainTextFilterRegexes []string `yaml:"regex-masking"` + IgnoredUserAgents []string `yaml:"ignored-user-agents"` + DisableRedaction bool `yaml:"no-redact" default:"false"` + HumanMaxEntriesDBSize string `yaml:"max-entries-db-size" default:"200MB"` + DryRun bool `yaml:"dry-run" default:"false"` + Workspace string `yaml:"workspace"` + EnforcePolicyFile string `yaml:"traffic-validation-file"` + ContractFile string `yaml:"contract"` + AskUploadConfirmation bool `yaml:"ask-upload-confirmation" default:"true"` + ApiServerResources shared.Resources `yaml:"api-server-resources"` + TapperResources shared.Resources `yaml:"tapper-resources"` + DaemonMode bool `yaml:"daemon" default:"false"` + NoPersistentVolumeClaim bool `yaml:"no-persistent-volume-claim" default:"false"` + Istio bool `yaml:"istio" default:"false"` } func (config *TapConfig) PodRegex() *regexp.Regexp { @@ -78,5 +84,9 @@ func (config *TapConfig) Validate() error { return errors.New(fmt.Sprintf("Can't run with both --%s and --%s flags", AnalysisTapName, WorkspaceTapName)) } + if config.NoPersistentVolumeClaim && !config.DaemonMode { + logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("the --set tap.no-persistent-volume-claim=true flag has no effect without the --%s flag, the claim will not be created anyway.", DaemonModeTapName)) + } + return nil } diff --git a/cli/go.mod b/cli/go.mod index e8233696f..88fed6c65 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -8,6 +8,7 @@ require ( github.com/getkin/kin-openapi v0.79.0 github.com/google/go-github/v37 v37.0.0 github.com/google/uuid v1.1.2 + github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 github.com/up9inc/mizu/shared v0.0.0 diff --git a/cli/mizu/consts.go b/cli/mizu/consts.go index 7c5ce60fa..73621a2ff 100644 --- a/cli/mizu/consts.go +++ b/cli/mizu/consts.go @@ -11,9 +11,12 @@ var ( GitCommitHash = "" // this var is overridden using ldflags in makefile when building BuildTimestamp = "" // this var is overridden using ldflags in makefile when building RBACVersion = "v1" + Platform = "" DaemonModePersistentVolumeSizeBufferBytes = int64(500 * 1000 * 1000) //500mb ) +const DEVENVVAR = "MIZU_DISABLE_TELEMTRY" + func GetMizuFolderPath() string { home, homeDirErr := os.UserHomeDir() if homeDirErr != nil { diff --git a/cli/mizu/fsUtils/mizuLogsUtils.go b/cli/mizu/fsUtils/mizuLogsUtils.go index 38295db9d..fe79936cc 100644 --- a/cli/mizu/fsUtils/mizuLogsUtils.go +++ b/cli/mizu/fsUtils/mizuLogsUtils.go @@ -78,6 +78,6 @@ func DumpLogs(ctx context.Context, provider *kubernetes.Provider, filePath strin logger.Log.Debugf("Successfully added file %s", GetLogFilePath()) } - logger.Log.Infof("You can find the zip file with all logs in %s\n", filePath) + logger.Log.Infof("You can find the zip file with all logs in %s", filePath) return nil } diff --git a/cli/mizu/version/versionCheck.go b/cli/mizu/version/versionCheck.go index 63f02ff2c..93d32d4d7 100644 --- a/cli/mizu/version/versionCheck.go +++ b/cli/mizu/version/versionCheck.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" "net/http" + "os" "runtime" "strings" "time" @@ -39,6 +40,10 @@ func CheckVersionCompatibility(apiServerProvider *apiserver.Provider) (bool, err } func CheckNewerVersion(versionChan chan string) { + if _, present := os.LookupEnv(mizu.DEVENVVAR); present { + versionChan <- "" + return + } logger.Log.Debugf("Checking for newer version...") start := time.Now() client := github.NewClient(nil) diff --git a/cli/telemetry/telemetry.go b/cli/telemetry/telemetry.go index 1bd10acfd..f89a7d877 100644 --- a/cli/telemetry/telemetry.go +++ b/cli/telemetry/telemetry.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net/http" + "os" "github.com/denisbrodbeck/machineid" "github.com/up9inc/mizu/cli/apiserver" @@ -62,6 +63,9 @@ func ReportAPICalls(apiProvider *apiserver.Provider) { } func shouldRunTelemetry() bool { + if _, present := os.LookupEnv(mizu.DEVENVVAR); present { + return false + } if !config.Config.Telemetry { return false } @@ -79,6 +83,7 @@ func sendTelemetry(telemetryType string, argsMap map[string]interface{}) error { argsMap["buildTimestamp"] = mizu.BuildTimestamp argsMap["branch"] = mizu.Branch argsMap["version"] = mizu.SemVer + argsMap["Platform"] = mizu.Platform if machineId, err := machineid.ProtectedID("mizu"); err == nil { argsMap["machineId"] = machineId diff --git a/cli/up9/provider.go b/cli/up9/provider.go new file mode 100644 index 000000000..63544db04 --- /dev/null +++ b/cli/up9/provider.go @@ -0,0 +1,31 @@ +package up9 + +import ( + "fmt" + "net/http" + "net/url" +) + +func IsTokenValid(tokenString string, envName string) bool { + whoAmIUrl, _ := url.Parse(fmt.Sprintf("https://trcc.%s/admin/whoami", envName)) + + req := &http.Request{ + Method: http.MethodGet, + URL: whoAmIUrl, + Header: map[string][]string{ + "Authorization": {fmt.Sprintf("bearer %s", tokenString)}, + }, + } + + response, err := http.DefaultClient.Do(req) + if err != nil { + return false + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return false + } + + return true +} diff --git a/debug.Dockerfile b/debug.Dockerfile index 2c91746d5..e063e887f 100644 --- a/debug.Dockerfile +++ b/debug.Dockerfile @@ -12,7 +12,7 @@ FROM golang:1.16-alpine AS builder # Set necessary environment variables needed for our image. ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64 -RUN apk add libpcap-dev gcc g++ make bash +RUN apk add libpcap-dev gcc g++ make bash perl-utils # Move to agent working directory (/agent-build). WORKDIR /app/agent-build @@ -23,7 +23,7 @@ COPY tap/go.mod tap/go.mod ../tap/ COPY tap/api/go.* ../tap/api/ RUN go mod download # cheap trick to make the build faster (As long as go.mod wasn't changes) -RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get +RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get ARG COMMIT_HASH ARG GIT_BRANCH @@ -36,17 +36,25 @@ COPY tap ../tap COPY agent . RUN go build -gcflags="all=-N -l" -o mizuagent . +# Download Basenine executable, verify the sha1sum and move it to a directory in $PATH +ADD https://github.com/up9inc/basenine/releases/download/v0.2.19/basenine_linux_amd64 ./basenine_linux_amd64 +ADD https://github.com/up9inc/basenine/releases/download/v0.2.19/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256 +RUN shasum -a 256 -c basenine_linux_amd64.sha256 +RUN chmod +x ./basenine_linux_amd64 + COPY devops/build_extensions_debug.sh .. RUN cd .. && /bin/bash build_extensions_debug.sh FROM golang:1.16-alpine -RUN apk add bash libpcap-dev tcpdump +RUN apk add bash libpcap-dev + WORKDIR /app # Copy binary and config files from /build to root folder of scratch container. COPY --from=builder ["/app/agent-build/mizuagent", "."] +COPY --from=builder ["/app/agent-build/basenine_linux_amd64", "/usr/local/bin/basenine"] COPY --from=builder ["/app/agent/build/extensions", "extensions"] COPY --from=site-build ["/app/ui-build/build", "site"] diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md new file mode 100644 index 000000000..ad9c64889 --- /dev/null +++ b/docs/CONFIGURATION.md @@ -0,0 +1,91 @@ +![Mizu: The API Traffic Viewer for Kubernetes](../assets/mizu-logo.svg) +# Configuration options for Mizu + +Mizu has many configuration options and flags that affect its behavior. Their values can be modified via command-line interface or via configuration file. + +The list below covers most useful configuration options. + +### Config file +Mizu behaviour can be modified via YAML configuration file located at `$HOME/.mizu/config.yaml`. + +Default values for the file can be viewed via `mizu config` command. + +### Applying config options via command line +To apply any configuration option via command line, use `--set` following by config option name and value, like in the following example: + +``` +mizu tap --set tap.dry-run=true +``` + +Please make sure to use full option name (`tap.dry-run` as opposed to `dry-run` only), incl. section (`tap`, in this example) + +## General section + +* `agent-image` - full path to Mizu container image, in format `full.path.to/your/image:tag`. Default value is set at compilation time to `gcr.io/up9-docker-hub/mizu/:` + +* `dump-logs` - if set to `true`, saves log files for all Mizu components (tapper, api-server, CLI) in a zip file under `$HOME/.mizu`. Default value is `false` + +* `image-pull-policy` - container image pull policy for Kubernetes, default value `Always`. Other accepted values are `Never` or `IfNotExist`. Please mind the implications when changing this. + +* `kube-config-path` - path to alternative kubeconfig file to use for all interactions with Kubernetes cluster. By default - `$HOME/.kubeconfig` + +* `mizu-resources-namespace` - Kubernetes namespace where all Mizu-related resources are created. Default value `mizu` + +* `telemetry` - report anonymous usage statistics. Default value `true` + +## section `tap` +* `namespaces` - list of namespace names, in which pods are tapped. Default value is empty, meaning only pods in the current namespace are tapped. Typically supplied as command line options. + +* `all-namespaces` - special flag indicating whether Mizu should search and tap pods, matching the regex, in all namespaces. Default is `false`. Please use with caution, tapping too many pods can affect resource consumption. + +* `daemon` - instructs Mizu whether to run daemon mode (where CLI command exits after launch, and tapper & api-server pods in Kubernetes continue to run without controlling CLI). Typically supplied as command-line option `--daemon`. Default valie is `false` + +* `dry-run` - if true, Mizu will print list of pods matching the supplied (or default) regex and exit without actually tapping the traffic. Default value is `false`. Typically supplied as command-line option `--dry-run` + +* `proxy-host` - IP address on which proxy to Mizu API service is launched; should be accessible at `proxy-host:gui-port`. Default value is `127.0.0.1` + +* `gui-port` - port on which Mizu GUI is accessible, default value is `8899` (stands for `8899/tcp`) + +* `regex` - regular expression used to match pods to tap, when no regex is given in the command line; default value is `.*`, which means `mizu tap` with no additional arguments is runnining as `mizu tap .*` (i.e. tap all pods found in current workspace) + +* `no-redact` - instructs Mizu whether to redact certain sensitive fields in the collected traffic. Default value is `false`, i.e. Mizu will replace sentitive data values with *REDACTED* placeholder. + +* `ignored-user-agents` - array of strings, describing HTTP *User-Agent* header values to be ignored. Useful to ignore Kubernetes healthcheck and other similar noisy periodic probes. Default value is empty. + +* `max-entries-db-size` - maximal size of traffic stored locally in the `mizu-api-server` pod. When this size is reached, older traffic is overwritten with new entries. Default value is `200MB` + + +### section `tap.api-server-resources` +Kubernetes request and limit values for the `mizu-api-server` pod. +Parameters and their default values are same as used natively in Kubernetes pods: + +``` + cpu-limit: 750m + memory-limit: 1Gi + cpu-requests: 50m + memory-requests: 50Mi +``` + +### section `tap.tapper-resources` +Kubernetes request and limit values for the `mizu-tapper` pods (launched via daemonset). +Parameters and their default values are same as used natively in Kubernetes pods: + +``` + cpu-limit: 750m + memory-limit: 1Gi + cpu-requests: 50m + memory-requests: 50Mi +``` + + +-- + +* `analsys` - enables advanced analysis of collected traffic in the UP9 coud platform. Default value is `false` + +* `upload-interval` - in the *analysis* mode, push traffic to UP9 cloud every `upload-interval` seconds. Default value is `10` seconds + +* `ask-upload-confirmation` - request user confirmation when uploading tapped data to UP9 cloud + + +## section `version` +* `debug`- print additional version and build information when `mizu version` command is invoked. Default value is `false`. diff --git a/docs/DAEMON_MODE.md b/docs/DAEMON_MODE.md new file mode 100644 index 000000000..ab28d7d12 --- /dev/null +++ b/docs/DAEMON_MODE.md @@ -0,0 +1,80 @@ +# Mizu daemon mode + +Mizu can be run detached from the cli using the daemon flag: `mizu tap --daemon`. This type of mizu instance will run +indefinitely in the cluster. + +Please note that daemon mode requires you to have RBAC creation permissions, see the [permissions](PERMISSIONS.md) +doc for more details. + +```bash +$ mizu tap "^ca.*" --daemon + Mizu will store up to 200MB of traffic, old traffic will be cleared once the limit is reached. + Tapping pods in namespaces "sock-shop" + Waiting for mizu to be ready... (may take a few minutes) + +carts-66c77f5fbb-fq65r + +catalogue-5f4cb7cf5-7zrmn + .. +``` + +## Stop mizu daemon + +To stop the detached mizu instance and clean all cluster side resources, run `mizu clean` + +```bash +$ mizu clean # mizu will continue running in cluster until clean is executed + Removing mizu resources +``` + +## Expose mizu web app + +Mizu could be exposed at a later stage in any of the following ways: + +### Using mizu view command + +In a machine that can access both the cluster and a browser, you can run `mizu view` command which creates a proxy. +Besides that, all the regular ways to expose k8s pods are valid. + +```bash +$ mizu view + Establishing connection to k8s cluster... + Mizu is available at http://localhost:8899 + ^C + .. +``` + +### Port Forward + +```bash +$ kubectl port-forward -n mizu deployment/mizu-api-server 8899:8899 +``` + +### NodePort + +```bash +$ kubectl expose -n mizu deployment mizu-api-server --name mizu-node-port --type NodePort --port 80 --target-port 8899 +``` + +Mizu's IP is the IP of any node (get the IP with `kubectl get nodes -o wide`) and the port is the target port of the new +service (`kubectl get services -n mizu mizu-node-port`). Note that this method will expose Mizu to public access if your +nodes are public. + +### LoadBalancer + +```bash +$ kubectl expose deployment -n mizu --port 80 --target-port 8899 mizu-api-server --type=LoadBalancer --name=mizu-lb + service/mizu-lb exposed + .. + +$ kubectl get services -n mizu + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + mizu-api-server ClusterIP 10.107.200.100 80/TCP 5m5s + mizu-lb LoadBalancer 10.107.200.101 34.77.120.116 80:30141/TCP 76s +``` + +Note that `LoadBalancer` services only work on supported clusters (usually cloud providers) and might incur extra costs + +If you changed the `mizu-resources-namespace` value, make sure the `-n mizu` flag of the `kubectl expose` command is +changed to the value of `mizu-resources-namespace` + +mizu will now be available both by running `mizu view` or by accessing the `EXTERNAL-IP` of the `mizu-lb` service +through your browser. diff --git a/docs/ISTIO.md b/docs/ISTIO.md new file mode 100644 index 000000000..bda897b65 --- /dev/null +++ b/docs/ISTIO.md @@ -0,0 +1,46 @@ +![Mizu: The API Traffic Viewer for Kubernetes](../assets/mizu-logo.svg) +# Istio mutual tls (mtls) with Mizu +This document describe how Mizu tapper handles workloads configured with mtls, making the internal traffic between services in a cluster to be encrypted. + +Besides Istio there are other service meshes that implement mtls. However, as of now Istio is the most used one, and this is why we are focusing on it. + +In order to create an Istio setup for development, follow those steps: +1. Deploy a sample application to a Kubernetes cluster, the sample application needs to make internal service to service calls +2. SSH to one of the nodes, and run `tcpdump` +3. Make sure you see the internal service to service calls in a plain text +4. Deploy Istio to the cluster - make sure it is attached to all pods of the sample application, and that it is configured with mtls (default) +5. Run `tcpdump` again, make sure you don't see the internal service to service calls in a plain text + +## The connection between Istio and Envoy +In order to implement its service mesh capabilities, [Istio](https://istio.io) use an [Envoy](https://www.envoyproxy.io) sidecar in front of every pod in the cluster. The Envoy is responsible for the mtls communication, and that's why we are focusing on Envoy proxy. + +In the future we might see more players in that field, then we'll have to either add support for each of them or go with a unified eBPF solution. + +## Network namespaces +A [linux network namespace](https://man7.org/linux/man-pages/man7/network_namespaces.7.html) is an isolation that limit the process view of the network. In the container world it used to isolate one container from another. In the Kubernetes world it used to isolate a pod from another. That means that two containers running on the same pod share the same network namespace. A container can reach a container in the same pod by accessing `localhost`. + +An Envoy proxy configured with mtls receives the inbound traffic directed to the pod, decrypts it and sends it via `localhost` to the target container. + +## Tapping mtls traffic +In order for Mizu to be able to see the decrypted traffic it needs to listen on the same network namespace of the target pod. Multiple threads of the same process can have different network namespaces. + +[gopacket](https://github.com/google/gopacket) uses [libpacp](https://github.com/the-tcpdump-group/libpcap) by default for capturing the traffic. Libpacap doesn't support network namespaces and we can't ask it to listen to traffic on a different namespace. However, we can change the network namespace of the calling thread and then start libpcap to see the traffic on a different namespace. + +## Finding the network namespace of a running process +The network namespace of a running process can be found in `/proc/PID/ns/net` link. Once we have this link, we can ask Linux to change the network namespace of a thread to this one. + +This mean that Mizu needs to have access to the `/proc` (procfs) of the running node. + +## Finding the network namespace of a running pod +In order for Mizu to be able to listen to mtls traffic, it needs to get the PIDs of the the running pods, filter them according to the user filters and then start listen to their internal network namespace traffic. + +There is no official way in Kubernetes to get from pod to PID. The CRI implementation purposefully doesn't force a pod to be a processes on the host. It can be a Virtual Machine as well like [Kata containers](https://katacontainers.io) + +While we can provide a solution for various CRIs (like Docker, Containerd and CRI-O) it's better to have a unified solution. In order to achieve that, Mizu scans all the processes in the host, and finds the Envoy processes using their `/proc/PID/exe` link. + +Once Mizu detects an Envoy process, it need to check whether this specific Envoy process is relevant according the user filters. The user filters are a list of `CLUSTER_IPS`. The tapper gets them via the `TapOpts.FilterAuthorities` list. + +Istio sends an `INSTANCE_IP` environment variable to every Envoy proxy process. By examining the Envoy process's environment variables we can see whether it's relevant or not. Examining a process environment variables is done by reading the `/proc/PID/envion` file. + +## Edge cases +The method we use to find Envoy processes and correlate them to the cluster IPs may be inaccurate in certain situations. If, for example, a user runs an Envoy process manually, and set its `INSTANCE_IP` environment variable to one of the `CLUSTER_IPS` the tapper gets, then Mizu will capture traffic for it. diff --git a/examples/roles/permissions-all-namespaces-daemon.yaml b/examples/roles/permissions-all-namespaces-daemon.yaml index 5a32eaaf5..99d36110e 100644 --- a/examples/roles/permissions-all-namespaces-daemon.yaml +++ b/examples/roles/permissions-all-namespaces-daemon.yaml @@ -7,15 +7,15 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch", "delete"] - - apiGroups: [ "" ] + - apiGroups: [ "apps" ] resources: [ "deployments" ] - verbs: [ "create", "delete" ] + verbs: [ "get", "create", "delete" ] - apiGroups: [""] resources: ["services"] verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: ["apps"] resources: ["daemonsets"] - verbs: ["create", "patch", "delete"] + verbs: ["get", "create", "patch", "delete", "list"] - apiGroups: [""] resources: ["namespaces"] verbs: ["get", "list", "watch", "create", "delete"] @@ -49,6 +49,9 @@ rules: - apiGroups: ["", "apps", "extensions"] resources: ["endpoints"] verbs: ["get", "list", "watch"] + - apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["list", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/examples/roles/permissions-all-namespaces-without-ip-resolution.yaml b/examples/roles/permissions-all-namespaces-without-ip-resolution.yaml index bced278e9..0a743e776 100644 --- a/examples/roles/permissions-all-namespaces-without-ip-resolution.yaml +++ b/examples/roles/permissions-all-namespaces-without-ip-resolution.yaml @@ -23,6 +23,9 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "create", "delete"] + - apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["list", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/examples/roles/permissions-all-namespaces.yaml b/examples/roles/permissions-all-namespaces.yaml index 097d276e0..7b8b065ee 100644 --- a/examples/roles/permissions-all-namespaces.yaml +++ b/examples/roles/permissions-all-namespaces.yaml @@ -46,6 +46,9 @@ rules: - apiGroups: ["", "apps", "extensions"] resources: ["endpoints"] verbs: ["get", "list", "watch"] +- apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["list", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/examples/roles/permissions-ns-daemon.yaml b/examples/roles/permissions-ns-daemon.yaml index c73513e8c..0ab880f11 100644 --- a/examples/roles/permissions-ns-daemon.yaml +++ b/examples/roles/permissions-ns-daemon.yaml @@ -8,7 +8,7 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch", "delete"] -- apiGroups: [ "" ] +- apiGroups: [ "apps" ] resources: [ "deployments" ] verbs: [ "get", "create", "delete" ] - apiGroups: [""] @@ -16,7 +16,7 @@ rules: verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: ["apps"] resources: ["daemonsets"] - verbs: ["get", "create", "patch", "delete"] + verbs: ["get", "create", "patch", "delete", "list"] - apiGroups: [""] resources: ["services/proxy"] verbs: ["get"] @@ -32,7 +32,7 @@ rules: - apiGroups: ["rbac.authorization.k8s.io"] resources: ["rolebindings"] verbs: ["get", "create", "delete"] -- apiGroups: ["apps", "extensions"] +- apiGroups: ["apps", "extensions", ""] resources: ["pods"] verbs: ["get", "list", "watch"] - apiGroups: ["apps", "extensions"] @@ -41,6 +41,9 @@ rules: - apiGroups: ["", "apps", "extensions"] resources: ["endpoints"] verbs: ["get", "list", "watch"] +- apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["list", "watch"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/examples/roles/permissions-ns-with-validation.yaml b/examples/roles/permissions-ns-with-validation.yaml index e2d6863ec..a3e3eceb2 100644 --- a/examples/roles/permissions-ns-with-validation.yaml +++ b/examples/roles/permissions-ns-with-validation.yaml @@ -38,6 +38,9 @@ rules: - apiGroups: ["", "apps", "extensions"] resources: ["endpoints"] verbs: ["get", "list", "watch"] +- apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["list", "watch"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/examples/roles/permissions-ns-without-ip-resolution.yaml b/examples/roles/permissions-ns-without-ip-resolution.yaml index ef14933a1..24bc0d822 100644 --- a/examples/roles/permissions-ns-without-ip-resolution.yaml +++ b/examples/roles/permissions-ns-without-ip-resolution.yaml @@ -20,6 +20,9 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "create", "delete"] +- apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["list", "watch"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/examples/roles/permissions-ns.yaml b/examples/roles/permissions-ns.yaml index 3af89afa0..6974ab50f 100644 --- a/examples/roles/permissions-ns.yaml +++ b/examples/roles/permissions-ns.yaml @@ -38,6 +38,9 @@ rules: - apiGroups: ["", "apps", "extensions"] resources: ["endpoints"] verbs: ["get", "list", "watch"] +- apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["list", "watch"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/shared/consts.go b/shared/consts.go index 79e4b84e5..97dc362d7 100644 --- a/shared/consts.go +++ b/shared/consts.go @@ -13,5 +13,7 @@ const ( ConfigFileName = "mizu-config.json" GoGCEnvVar = "GOGC" DefaultApiServerPort = 8899 - DebugModeEnvVar = "MIZU_DEBUG" + LogLevelEnvVar = "LOG_LEVEL" + BasenineHost = "localhost" + BaseninePort = "9099" ) diff --git a/shared/kubernetes/eventWatchHelper.go b/shared/kubernetes/eventWatchHelper.go new file mode 100644 index 000000000..332fdd4e6 --- /dev/null +++ b/shared/kubernetes/eventWatchHelper.go @@ -0,0 +1,52 @@ +package kubernetes + +import ( + "context" + "regexp" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" +) + +type EventWatchHelper struct { + kubernetesProvider *Provider + NameRegexFilter *regexp.Regexp + Kind string +} + +func NewEventWatchHelper(kubernetesProvider *Provider, NameRegexFilter *regexp.Regexp, kind string) *EventWatchHelper { + return &EventWatchHelper{ + kubernetesProvider: kubernetesProvider, + NameRegexFilter: NameRegexFilter, + Kind: kind, + } +} + +// Implements the EventFilterer Interface +func (wh *EventWatchHelper) Filter(wEvent *WatchEvent) (bool, error) { + event, err := wEvent.ToEvent() + if err != nil { + return false, nil + } + + if !wh.NameRegexFilter.MatchString(event.Name) { + return false, nil + } + + if strings.ToLower(event.Regarding.Kind) != strings.ToLower(wh.Kind) { + return false, nil + } + + return true, nil +} + +// Implements the WatchCreator Interface +func (wh *EventWatchHelper) NewWatcher(ctx context.Context, namespace string) (watch.Interface, error) { + watcher, err := wh.kubernetesProvider.clientSet.EventsV1().Events(namespace).Watch(ctx, metav1.ListOptions{Watch: true}) + if err != nil { + return nil, err + } + + return watcher, nil +} diff --git a/shared/kubernetes/mizuTapperSyncer.go b/shared/kubernetes/mizuTapperSyncer.go index d6b81387a..31dba52d1 100644 --- a/shared/kubernetes/mizuTapperSyncer.go +++ b/shared/kubernetes/mizuTapperSyncer.go @@ -3,6 +3,7 @@ package kubernetes import ( "context" "fmt" + "github.com/op/go-logging" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/debounce" "github.com/up9inc/mizu/shared/logger" @@ -15,18 +16,22 @@ import ( const updateTappersDelay = 5 * time.Second type TappedPodChangeEvent struct { - Added []core.Pod - Removed []core.Pod + Added []core.Pod + Removed []core.Pod + ExpectedTapperAmount int } // MizuTapperSyncer uses a k8s pod watch to update tapper daemonsets when targeted pods are removed or created type MizuTapperSyncer struct { - context context.Context - CurrentlyTappedPods []core.Pod - config TapperSyncerConfig - kubernetesProvider *Provider - TapPodChangesOut chan TappedPodChangeEvent - ErrorOut chan K8sTapManagerError + startTime time.Time + context context.Context + CurrentlyTappedPods []core.Pod + config TapperSyncerConfig + kubernetesProvider *Provider + TapPodChangesOut chan TappedPodChangeEvent + TapperStatusChangedOut chan shared.TapperStatus + ErrorOut chan K8sTapManagerError + nodeToTappedPodMap map[string][]core.Pod } type TapperSyncerConfig struct { @@ -36,20 +41,23 @@ type TapperSyncerConfig struct { AgentImage string TapperResources shared.Resources ImagePullPolicy core.PullPolicy - DumpLogs bool + LogLevel logging.Level IgnoredUserAgents []string MizuApiFilteringOptions api.TrafficFilteringOptions MizuServiceAccountExists bool + Istio bool } -func CreateAndStartMizuTapperSyncer(ctx context.Context, kubernetesProvider *Provider, config TapperSyncerConfig) (*MizuTapperSyncer, error) { +func CreateAndStartMizuTapperSyncer(ctx context.Context, kubernetesProvider *Provider, config TapperSyncerConfig, startTime time.Time) (*MizuTapperSyncer, error) { syncer := &MizuTapperSyncer{ - context: ctx, - CurrentlyTappedPods: make([]core.Pod, 0), - config: config, - kubernetesProvider: kubernetesProvider, - TapPodChangesOut: make(chan TappedPodChangeEvent, 100), - ErrorOut: make(chan K8sTapManagerError, 100), + startTime: startTime.Truncate(time.Second), // Round down because k8s CreationTimestamp is given in 1 sec resolution. + context: ctx, + CurrentlyTappedPods: make([]core.Pod, 0), + config: config, + kubernetesProvider: kubernetesProvider, + TapPodChangesOut: make(chan TappedPodChangeEvent, 100), + TapperStatusChangedOut: make(chan shared.TapperStatus, 100), + ErrorOut: make(chan K8sTapManagerError, 100), } if err, _ := syncer.updateCurrentlyTappedPods(); err != nil { @@ -61,11 +69,75 @@ func CreateAndStartMizuTapperSyncer(ctx context.Context, kubernetesProvider *Pro } go syncer.watchPodsForTapping() + go syncer.watchTapperEvents() return syncer, nil } +func (tapperSyncer *MizuTapperSyncer) watchTapperEvents() { + mizuResourceRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", TapperPodName)) + eventWatchHelper := NewEventWatchHelper(tapperSyncer.kubernetesProvider, mizuResourceRegex, "pod") + eventChan, errorChan := FilteredWatch(tapperSyncer.context, eventWatchHelper, []string{tapperSyncer.config.MizuResourcesNamespace}, eventWatchHelper) + + for { + select { + case wEvent, ok := <-eventChan: + if !ok { + eventChan = nil + continue + } + + event, err := wEvent.ToEvent() + if err != nil { + logger.Log.Errorf(fmt.Sprintf("Error parsing Mizu resource event: %+v", err)) + } + + if tapperSyncer.startTime.After(event.CreationTimestamp.Time) { + continue + } + + logger.Log.Debugf( + fmt.Sprintf("Watching tapper events loop, event %s, time: %v, resource: %s (%s), reason: %s, note: %s", + event.Name, + event.CreationTimestamp.Time, + event.Regarding.Name, + event.Regarding.Kind, + event.Reason, + event.Note)) + + pod, err1 := tapperSyncer.kubernetesProvider.GetPod(tapperSyncer.context, tapperSyncer.config.MizuResourcesNamespace, event.Regarding.Name) + if err1 != nil { + logger.Log.Debugf(fmt.Sprintf("Failed to get tapper pod %s", event.Regarding.Name)) + continue + } + + nodeName := "" + if event.Reason != "FailedScheduling" { + nodeName = pod.Spec.NodeName + } else { + nodeName = pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchFields[0].Values[0] + } + + taperStatus := shared.TapperStatus{TapperName: pod.Name, NodeName: nodeName, Status: event.Reason} + tapperSyncer.TapperStatusChangedOut <- taperStatus + + case err, ok := <-errorChan: + if !ok { + errorChan = nil + continue + } + + logger.Log.Errorf("Watching tapper events loop, error: %+v", err) + + case <-tapperSyncer.context.Done(): + logger.Log.Debugf("Watching tapper events loop, ctx done") + return + } + } +} + func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() { - added, modified, removed, errorChan := FilteredWatch(tapperSyncer.context, tapperSyncer.kubernetesProvider, tapperSyncer.config.TargetNamespaces, &tapperSyncer.config.PodFilterRegex) + podWatchHelper := NewPodWatchHelper(tapperSyncer.kubernetesProvider, &tapperSyncer.config.PodFilterRegex) + eventChan, errorChan := FilteredWatch(tapperSyncer.context, podWatchHelper, tapperSyncer.config.TargetNamespaces, podWatchHelper) restartTappers := func() { err, changeFound := tapperSyncer.updateCurrentlyTappedPods() @@ -91,37 +163,40 @@ func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() { for { select { - case pod, ok := <-added: + case wEvent, ok := <-eventChan: if !ok { - added = nil + eventChan = nil continue } - logger.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace) - restartTappersDebouncer.SetOn() - case pod, ok := <-removed: - if !ok { - removed = nil + pod, err := wEvent.ToPod() + if err != nil { + tapperSyncer.handleErrorInWatchLoop(err, restartTappersDebouncer) continue } - logger.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace) - restartTappersDebouncer.SetOn() - case pod, ok := <-modified: - if !ok { - modified = nil - continue - } - - logger.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP) - // Act only if the modified pod has already obtained an IP address. - // After filtering for IPs, on a normal pod restart this includes the following events: - // - Pod deletion - // - Pod reaches start state - // - Pod reaches ready state - // Ready/unready transitions might also trigger this event. - if pod.Status.PodIP != "" { + switch wEvent.Type { + case EventAdded: + logger.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace) restartTappersDebouncer.SetOn() + case EventDeleted: + logger.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace) + restartTappersDebouncer.SetOn() + case EventModified: + logger.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP) + // Act only if the modified pod has already obtained an IP address. + // After filtering for IPs, on a normal pod restart this includes the following events: + // - Pod deletion + // - Pod reaches start state + // - Pod reaches ready state + // Ready/unready transitions might also trigger this event. + if pod.Status.PodIP != "" { + restartTappersDebouncer.SetOn() + } + case EventBookmark: + break + case EventError: + break } case err, ok := <-errorChan: if !ok { @@ -129,12 +204,8 @@ func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() { continue } - logger.Log.Debugf("Watching pods loop, got error %v, stopping `restart tappers debouncer`", err) - restartTappersDebouncer.Cancel() - tapperSyncer.ErrorOut <- K8sTapManagerError{ - OriginalError: err, - TapManagerReason: TapManagerPodWatchError, - } + tapperSyncer.handleErrorInWatchLoop(err, restartTappersDebouncer) + continue case <-tapperSyncer.context.Done(): logger.Log.Debugf("Watching pods loop, context done, stopping `restart tappers debouncer`") @@ -145,6 +216,15 @@ func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() { } } +func (tapperSyncer *MizuTapperSyncer) handleErrorInWatchLoop(err error, restartTappersDebouncer *debounce.Debouncer) { + logger.Log.Debugf("Watching pods loop, got error %v, stopping `restart tappers debouncer`", err) + restartTappersDebouncer.Cancel() + tapperSyncer.ErrorOut <- K8sTapManagerError{ + OriginalError: err, + TapManagerReason: TapManagerPodWatchError, + } +} + func (tapperSyncer *MizuTapperSyncer) updateCurrentlyTappedPods() (err error, changesFound bool) { if matchingPods, err := tapperSyncer.kubernetesProvider.ListAllRunningPodsMatchingRegex(tapperSyncer.context, &tapperSyncer.config.PodFilterRegex, tapperSyncer.config.TargetNamespaces); err != nil { return err, false @@ -159,9 +239,11 @@ func (tapperSyncer *MizuTapperSyncer) updateCurrentlyTappedPods() (err error, ch } if len(addedPods) > 0 || len(removedPods) > 0 { tapperSyncer.CurrentlyTappedPods = podsToTap + tapperSyncer.nodeToTappedPodMap = GetNodeHostToTappedPodsMap(tapperSyncer.CurrentlyTappedPods) tapperSyncer.TapPodChangesOut <- TappedPodChangeEvent{ - Added: addedPods, - Removed: removedPods, + Added: addedPods, + Removed: removedPods, + ExpectedTapperAmount: len(tapperSyncer.nodeToTappedPodMap), } return nil, true } @@ -170,9 +252,7 @@ func (tapperSyncer *MizuTapperSyncer) updateCurrentlyTappedPods() (err error, ch } func (tapperSyncer *MizuTapperSyncer) updateMizuTappers() error { - nodeToTappedPodIPMap := GetNodeHostToTappedPodIpsMap(tapperSyncer.CurrentlyTappedPods) - - if len(nodeToTappedPodIPMap) > 0 { + if len(tapperSyncer.nodeToTappedPodMap) > 0 { var serviceAccountName string if tapperSyncer.config.MizuServiceAccountExists { serviceAccountName = ServiceAccountName @@ -187,16 +267,17 @@ func (tapperSyncer *MizuTapperSyncer) updateMizuTappers() error { tapperSyncer.config.AgentImage, TapperPodName, fmt.Sprintf("%s.%s.svc.cluster.local", ApiServerPodName, tapperSyncer.config.MizuResourcesNamespace), - nodeToTappedPodIPMap, + tapperSyncer.nodeToTappedPodMap, serviceAccountName, tapperSyncer.config.TapperResources, tapperSyncer.config.ImagePullPolicy, tapperSyncer.config.MizuApiFilteringOptions, - tapperSyncer.config.DumpLogs, + tapperSyncer.config.LogLevel, + tapperSyncer.config.Istio, ); err != nil { return err } - logger.Log.Debugf("Successfully created %v tappers", len(nodeToTappedPodIPMap)) + logger.Log.Debugf("Successfully created %v tappers", len(tapperSyncer.nodeToTappedPodMap)) } else { if err := tapperSyncer.kubernetesProvider.RemoveDaemonSet(tapperSyncer.context, tapperSyncer.config.MizuResourcesNamespace, TapperDaemonSetName); err != nil { return err diff --git a/shared/kubernetes/podWatchHelper.go b/shared/kubernetes/podWatchHelper.go new file mode 100644 index 000000000..771964466 --- /dev/null +++ b/shared/kubernetes/podWatchHelper.go @@ -0,0 +1,45 @@ +package kubernetes + +import ( + "context" + "regexp" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" +) + +type PodWatchHelper struct { + kubernetesProvider *Provider + NameRegexFilter *regexp.Regexp +} + +func NewPodWatchHelper(kubernetesProvider *Provider, NameRegexFilter *regexp.Regexp) *PodWatchHelper { + return &PodWatchHelper{ + kubernetesProvider: kubernetesProvider, + NameRegexFilter: NameRegexFilter, + } +} + +// Implements the EventFilterer Interface +func (wh *PodWatchHelper) Filter(wEvent *WatchEvent) (bool, error) { + pod, err := wEvent.ToPod() + if err != nil { + return false, nil + } + + if !wh.NameRegexFilter.MatchString(pod.Name) { + return false, nil + } + + return true, nil +} + +// Implements the WatchCreator Interface +func (wh *PodWatchHelper) NewWatcher(ctx context.Context, namespace string) (watch.Interface, error) { + watcher, err := wh.kubernetesProvider.clientSet.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{Watch: true}) + if err != nil { + return nil, err + } + + return watcher, nil +} diff --git a/shared/kubernetes/provider.go b/shared/kubernetes/provider.go index 1adda9858..a1a0195d1 100644 --- a/shared/kubernetes/provider.go +++ b/shared/kubernetes/provider.go @@ -6,11 +6,16 @@ import ( "encoding/json" "errors" "fmt" + "io" + "net/url" + "path/filepath" + "regexp" + + "github.com/op/go-logging" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" "github.com/up9inc/mizu/shared/semver" "github.com/up9inc/mizu/tap/api" - "io" v1 "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" @@ -31,9 +36,6 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" watchtools "k8s.io/client-go/tools/watch" - "net/url" - "path/filepath" - "regexp" ) type Provider struct { @@ -45,6 +47,8 @@ type Provider struct { const ( fieldManagerName = "mizu-manager" + procfsVolumeName = "proc" + procfsMountPath = "/hostproc" ) func NewProvider(kubeConfigPath string) (*Provider, error) { @@ -150,14 +154,6 @@ func (provider *Provider) WaitUtilNamespaceDeleted(ctx context.Context, name str return err } -func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) watch.Interface { - watcher, err := provider.clientSet.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{Watch: true}) - if err != nil { - panic(err.Error()) - } - return watcher -} - func (provider *Provider) CreateNamespace(ctx context.Context, name string) (*core.Namespace, error) { namespaceSpec := &core.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -177,7 +173,7 @@ type ApiServerOptions struct { MaxEntriesDBSizeBytes int64 Resources shared.Resources ImagePullPolicy core.PullPolicy - DumpLogs bool + LogLevel logging.Level } func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, mountVolumeClaim bool, volumeClaimName string) (*core.Pod, error) { @@ -239,22 +235,15 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun }, }) volumeMounts = append(volumeMounts, core.VolumeMount{ - Name: volumeClaimName, - MountPath: shared.DataDirPath, + Name: volumeClaimName, + MountPath: shared.DataDirPath, }) } - port := intstr.FromInt(shared.DefaultApiServerPort) - - debugMode := "" - if opts.DumpLogs { - debugMode = "1" - } - pod := &core.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: opts.PodName, - Labels: map[string]string{"app": opts.PodName}, + Name: opts.PodName, + Labels: map[string]string{"app": opts.PodName}, }, Spec: core.PodSpec{ Containers: []core.Container{ @@ -262,16 +251,16 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun Name: opts.PodName, Image: opts.PodImage, ImagePullPolicy: opts.ImagePullPolicy, - VolumeMounts: volumeMounts, - Command: command, + VolumeMounts: volumeMounts, + Command: command, Env: []core.EnvVar{ { Name: shared.SyncEntriesConfigEnvVar, Value: string(marshaledSyncEntriesConfig), }, { - Name: shared.DebugModeEnvVar, - Value: debugMode, + Name: shared.LogLevelEnvVar, + Value: opts.LogLevel.String(), }, }, Resources: core.ResourceRequirements{ @@ -284,28 +273,9 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun "memory": memRequests, }, }, - ReadinessProbe: &core.Probe{ - Handler: core.Handler{ - TCPSocket: &core.TCPSocketAction{ - Port: port, - }, - }, - InitialDelaySeconds: 5, - PeriodSeconds: 10, - }, - LivenessProbe: &core.Probe{ - Handler: core.Handler{ - HTTPGet: &core.HTTPGetAction{ - Path: "/echo", - Port: port, - }, - }, - InitialDelaySeconds: 5, - PeriodSeconds: 10, - }, }, }, - Volumes: volumes, + Volumes: volumes, DNSPolicy: core.DNSClusterFirstWithHostNet, TerminationGracePeriodSeconds: new(int64), }, @@ -318,7 +288,6 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun return pod, nil } - func (provider *Provider) CreatePod(ctx context.Context, namespace string, podSpec *core.Pod) (*core.Pod, error) { return provider.clientSet.CoreV1().Pods(namespace).Create(ctx, podSpec, metav1.CreateOptions{}) } @@ -333,14 +302,14 @@ func (provider *Provider) CreateDeployment(ctx context.Context, namespace string } deployment := &v1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: deploymentName, + Name: deploymentName, }, Spec: v1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"app": podSpec.ObjectMeta.Labels["app"]}, }, - Template: *podTemplate, - Strategy: v1.DeploymentStrategy{}, + Template: *podTemplate, + Strategy: v1.DeploymentStrategy{}, }, } return provider.clientSet.AppsV1().Deployments(namespace).Create(ctx, deployment, metav1.CreateOptions{}) @@ -349,7 +318,7 @@ func (provider *Provider) CreateDeployment(ctx context.Context, namespace string func (provider *Provider) CreateService(ctx context.Context, namespace string, serviceName string, appLabelValue string) (*core.Service, error) { service := core.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, + Name: serviceName, }, Spec: core.ServiceSpec{ Ports: []core.ServicePort{{TargetPort: intstr.FromInt(shared.DefaultApiServerPort), Port: 80}}, @@ -361,8 +330,8 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s } func (provider *Provider) DoesServicesExist(ctx context.Context, namespace string, name string) (bool, error) { - resource, err := provider.clientSet.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{}) - return provider.doesResourceExist(resource, err) + serviceResource, err := provider.clientSet.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{}) + return provider.doesResourceExist(serviceResource, err) } func (provider *Provider) doesResourceExist(resource interface{}, err error) (bool, error) { @@ -381,8 +350,8 @@ func (provider *Provider) doesResourceExist(resource interface{}, err error) (bo func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, serviceAccountName string, clusterRoleName string, clusterRoleBindingName string, version string) error { serviceAccount := &core.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ - Name: serviceAccountName, - Labels: map[string]string{"mizu-cli-version": version}, + Name: serviceAccountName, + Labels: map[string]string{"mizu-cli-version": version}, }, } clusterRole := &rbac.ClusterRole{ @@ -434,8 +403,8 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context, namespace string, serviceAccountName string, roleName string, roleBindingName string, version string) error { serviceAccount := &core.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ - Name: serviceAccountName, - Labels: map[string]string{"mizu-cli-version": version}, + Name: serviceAccountName, + Labels: map[string]string{"mizu-cli-version": version}, }, } role := &rbac.Role{ @@ -496,6 +465,11 @@ func (provider *Provider) CreateDaemonsetRBAC(ctx context.Context, namespace str Resources: []string{"daemonsets"}, Verbs: []string{"patch", "get", "list", "create", "delete"}, }, + { + APIGroups: []string{"events.k8s.io"}, + Resources: []string{"events"}, + Verbs: []string{"list", "watch"}, + }, }, } roleBinding := &rbac.RoleBinding{ @@ -582,6 +556,11 @@ func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, return provider.handleRemovalError(err) } +func (provider *Provider) RemovePersistentVolumeClaim(ctx context.Context, namespace string, volumeClaimName string) error { + err := provider.clientSet.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, volumeClaimName, metav1.DeleteOptions{}) + return provider.handleRemovalError(err) +} + func (provider *Provider) handleRemovalError(err error) error { // Ignore NotFound - There is nothing to delete. // Ignore Forbidden - Assume that a user could not have created the resource in the first place. @@ -608,7 +587,7 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: configMapName, + Name: configMapName, }, Data: configMapData, } @@ -618,14 +597,14 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, return nil } -func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, dumpLogs bool) error { - logger.Log.Debugf("Applying %d tapper daemon sets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName) +func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodMap map[string][]core.Pod, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, logLevel logging.Level, istio bool) error { + logger.Log.Debugf("Applying %d tapper daemon sets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodMap), namespace, daemonSetName, podImage, tapperPodName) - if len(nodeToTappedPodIPMap) == 0 { + if len(nodeToTappedPodMap) == 0 { return fmt.Errorf("daemon set %s must tap at least 1 pod", daemonSetName) } - nodeToTappedPodIPMapJsonStr, err := json.Marshal(nodeToTappedPodIPMap) + nodeToTappedPodMapJsonStr, err := json.Marshal(nodeToTappedPodMap) if err != nil { return err } @@ -643,21 +622,30 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac "--nodefrag", } - debugMode := "" - if dumpLogs { - debugMode = "1" + if istio { + mizuCmd = append(mizuCmd, "--procfs", procfsMountPath, "--istio") } agentContainer := applyconfcore.Container() agentContainer.WithName(tapperPodName) agentContainer.WithImage(podImage) agentContainer.WithImagePullPolicy(imagePullPolicy) - agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(true)) + + caps := applyconfcore.Capabilities().WithDrop("ALL").WithAdd("NET_RAW").WithAdd("NET_ADMIN") + + if istio { + caps = caps.WithAdd("SYS_ADMIN") // for reading /proc/PID/net/ns + caps = caps.WithAdd("SYS_PTRACE") // for setting netns to other process + caps = caps.WithAdd("DAC_OVERRIDE") // for reading /proc/PID/environ + } + + agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithCapabilities(caps)) + agentContainer.WithCommand(mizuCmd...) agentContainer.WithEnv( - applyconfcore.EnvVar().WithName(shared.DebugModeEnvVar).WithValue(debugMode), + applyconfcore.EnvVar().WithName(shared.LogLevelEnvVar).WithValue(logLevel.String()), applyconfcore.EnvVar().WithName(shared.HostModeEnvVar).WithValue("1"), - applyconfcore.EnvVar().WithName(shared.TappedAddressesPerNodeDictEnvVar).WithValue(string(nodeToTappedPodIPMapJsonStr)), + applyconfcore.EnvVar().WithName(shared.TappedAddressesPerNodeDictEnvVar).WithValue(string(nodeToTappedPodMapJsonStr)), applyconfcore.EnvVar().WithName(shared.GoGCEnvVar).WithValue("12800"), applyconfcore.EnvVar().WithName(shared.MizuFilteringOptionsEnvVar).WithValue(string(mizuApiFilteringOptionsJsonStr)), ) @@ -695,8 +683,8 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac agentResources := applyconfcore.ResourceRequirements().WithRequests(agentResourceRequests).WithLimits(agentResourceLimits) agentContainer.WithResources(agentResources) - nodeNames := make([]string, 0, len(nodeToTappedPodIPMap)) - for nodeName := range nodeToTappedPodIPMap { + nodeNames := make([]string, 0, len(nodeToTappedPodMap)) + for nodeName := range nodeToTappedPodMap { nodeNames = append(nodeNames, nodeName) } nodeSelectorRequirement := applyconfcore.NodeSelectorRequirement() @@ -719,6 +707,14 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac noScheduleToleration.WithOperator(core.TolerationOpExists) noScheduleToleration.WithEffect(core.TaintEffectNoSchedule) + // Host procfs is needed inside the container because we need access to + // the network namespaces of processes on the machine. + // + procfsVolume := applyconfcore.Volume() + procfsVolume.WithName(procfsVolumeName).WithHostPath(applyconfcore.HostPathVolumeSource().WithPath("/proc")) + volumeMount := applyconfcore.VolumeMount().WithName(procfsVolumeName).WithMountPath(procfsMountPath).WithReadOnly(true) + agentContainer.WithVolumeMounts(volumeMount) + volumeName := ConfigMapName configMapVolume := applyconfcore.VolumeApplyConfiguration{ Name: &volumeName, @@ -747,7 +743,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac podSpec.WithContainers(agentContainer) podSpec.WithAffinity(affinity) podSpec.WithTolerations(noExecuteToleration, noScheduleToleration) - podSpec.WithVolumes(&configMapVolume) + podSpec.WithVolumes(&configMapVolume, procfsVolume) podTemplate := applyconfcore.PodTemplateSpec() podTemplate.WithLabels(map[string]string{"app": tapperPodName}) @@ -763,10 +759,10 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac return err } -func (provider *Provider) ListAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespaces []string) ([]core.Pod, error) { +func (provider *Provider) listPodsImpl(ctx context.Context, regex *regexp.Regexp, namespaces []string, listOptions metav1.ListOptions) ([]core.Pod, error) { var pods []core.Pod for _, namespace := range namespaces { - namespacePods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) + namespacePods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, listOptions) if err != nil { return nil, fmt.Errorf("failed to get pods in ns: [%s], %w", namespace, err) } @@ -783,6 +779,14 @@ func (provider *Provider) ListAllPodsMatchingRegex(ctx context.Context, regex *r return matchingPods, nil } +func (provider *Provider) ListAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespaces []string) ([]core.Pod, error) { + return provider.listPodsImpl(ctx, regex, namespaces, metav1.ListOptions{}) +} + +func (provider *Provider) GetPod(ctx context.Context, namespaces string, podName string) (*core.Pod, error) { + return provider.clientSet.CoreV1().Pods(namespaces).Get(ctx, podName, metav1.GetOptions{}) +} + func (provider *Provider) ListAllRunningPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespaces []string) ([]core.Pod, error) { pods, err := provider.ListAllPodsMatchingRegex(ctx, regex, namespaces) if err != nil { @@ -842,9 +846,9 @@ func (provider *Provider) CreatePersistentVolumeClaim(ctx context.Context, names ObjectMeta: metav1.ObjectMeta{ Name: volumeClaimName, }, - Spec: core.PersistentVolumeClaimSpec{ - AccessModes: []core.PersistentVolumeAccessMode{core.ReadWriteOnce}, - Resources: core.ResourceRequirements{ + Spec: core.PersistentVolumeClaimSpec{ + AccessModes: []core.PersistentVolumeAccessMode{core.ReadWriteOnce}, + Resources: core.ResourceRequirements{ Limits: core.ResourceList{ core.ResourceStorage: *sizeLimitQuantity, }, @@ -858,10 +862,6 @@ func (provider *Provider) CreatePersistentVolumeClaim(ctx context.Context, names return provider.clientSet.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, volumeClaim, metav1.CreateOptions{}) } -func (provider *Provider) RemovePersistentVolumeClaim(ctx context.Context, namespace string, volumeClaimName string) error { - return provider.clientSet.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, volumeClaimName, metav1.DeleteOptions{}) -} - func getClientSet(config *restclient.Config) (*kubernetes.Clientset, error) { clientSet, err := kubernetes.NewForConfig(config) if err != nil { diff --git a/shared/kubernetes/proxy.go b/shared/kubernetes/proxy.go index cfb69e4db..138bce331 100644 --- a/shared/kubernetes/proxy.go +++ b/shared/kubernetes/proxy.go @@ -28,9 +28,9 @@ func StartProxy(kubernetesProvider *Provider, proxyHost string, mizuPort uint16, return err } mux := http.NewServeMux() - mux.Handle(k8sProxyApiPrefix, proxyHandler) + mux.Handle(k8sProxyApiPrefix, getRerouteHttpHandlerMizuAPI(proxyHandler, mizuNamespace, mizuServiceName)) mux.Handle("/static/", getRerouteHttpHandlerMizuStatic(proxyHandler, mizuNamespace, mizuServiceName)) - mux.Handle("/mizu/", getRerouteHttpHandlerMizuAPI(proxyHandler, mizuNamespace, mizuServiceName)) + l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", proxyHost, int(mizuPort))) if err != nil { @@ -45,16 +45,21 @@ func StartProxy(kubernetesProvider *Provider, proxyHost string, mizuPort uint16, } func getMizuApiServerProxiedHostAndPath(mizuNamespace string, mizuServiceName string) string { - return fmt.Sprintf("/api/v1/namespaces/%s/services/%s:%d/proxy/", mizuNamespace, mizuServiceName, mizuServicePort) + return fmt.Sprintf("/api/v1/namespaces/%s/services/%s:%d/proxy", mizuNamespace, mizuServiceName, mizuServicePort) } func GetMizuApiServerProxiedHostAndPath(mizuPort uint16) string { - return fmt.Sprintf("localhost:%d/mizu", mizuPort) + return fmt.Sprintf("localhost:%d", mizuPort) } func getRerouteHttpHandlerMizuAPI(proxyHandler http.Handler, mizuNamespace string, mizuServiceName string) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - r.URL.Path = strings.Replace(r.URL.Path, "/mizu/", getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName), 1) + proxiedPath := getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName) + + //avoid redirecting several times + if !strings.Contains(r.URL.Path, proxiedPath) { + r.URL.Path = fmt.Sprintf("%s%s", getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName), r.URL.Path) + } proxyHandler.ServeHTTP(w, r) }) } diff --git a/shared/kubernetes/utils.go b/shared/kubernetes/utils.go index 6a020e259..39871e269 100644 --- a/shared/kubernetes/utils.go +++ b/shared/kubernetes/utils.go @@ -1,22 +1,38 @@ package kubernetes import ( + "regexp" + "github.com/up9inc/mizu/shared" core "k8s.io/api/core/v1" - "regexp" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func GetNodeHostToTappedPodIpsMap(tappedPods []core.Pod) map[string][]string { - nodeToTappedPodIPMap := make(map[string][]string, 0) +func GetNodeHostToTappedPodsMap(tappedPods []core.Pod) map[string][]core.Pod { + nodeToTappedPodMap := make(map[string][]core.Pod, 0) for _, pod := range tappedPods { - existingList := nodeToTappedPodIPMap[pod.Spec.NodeName] + minimizedPod := getMinimizedPod(pod) + + existingList := nodeToTappedPodMap[pod.Spec.NodeName] if existingList == nil { - nodeToTappedPodIPMap[pod.Spec.NodeName] = []string{pod.Status.PodIP} + nodeToTappedPodMap[pod.Spec.NodeName] = []core.Pod{minimizedPod} } else { - nodeToTappedPodIPMap[pod.Spec.NodeName] = append(nodeToTappedPodIPMap[pod.Spec.NodeName], pod.Status.PodIP) + nodeToTappedPodMap[pod.Spec.NodeName] = append(nodeToTappedPodMap[pod.Spec.NodeName], minimizedPod) } } - return nodeToTappedPodIPMap + return nodeToTappedPodMap +} + +func getMinimizedPod(fullPod core.Pod) core.Pod { + return core.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fullPod.Name, + }, + Status: v1.PodStatus{ + PodIP: fullPod.Status.PodIP, + }, + } } func excludeMizuPods(pods []core.Pod) []core.Pod { @@ -57,11 +73,10 @@ func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod { return missingPods } - func GetPodInfosForPods(pods []core.Pod) []shared.PodInfo { podInfos := make([]shared.PodInfo, 0) for _, pod := range pods { - podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace}) + podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace, NodeName: pod.Spec.NodeName}) } return podInfos } diff --git a/shared/kubernetes/watch.go b/shared/kubernetes/watch.go index 9bd47ebe4..6acc3ef7f 100644 --- a/shared/kubernetes/watch.go +++ b/shared/kubernetes/watch.go @@ -6,19 +6,22 @@ import ( "fmt" "github.com/up9inc/mizu/shared/debounce" "github.com/up9inc/mizu/shared/logger" - "regexp" "sync" "time" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/watch" ) -func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetNamespaces []string, podFilter *regexp.Regexp) (chan *corev1.Pod, chan *corev1.Pod, chan *corev1.Pod, chan error) { - addedChan := make(chan *corev1.Pod) - modifiedChan := make(chan *corev1.Pod) - removedChan := make(chan *corev1.Pod) +type EventFilterer interface { + Filter(*WatchEvent) (bool, error) +} + +type WatchCreator interface { + NewWatcher(ctx context.Context, namespace string) (watch.Interface, error) +} + +func FilteredWatch(ctx context.Context, watcherCreator WatchCreator, targetNamespaces []string, filterer EventFilterer) (<-chan *WatchEvent, <-chan error) { + eventChan := make(chan *WatchEvent) errorChan := make(chan error) var wg sync.WaitGroup @@ -31,8 +34,13 @@ func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetName watchRestartDebouncer := debounce.NewDebouncer(1 * time.Minute, func() {}) for { - watcher := kubernetesProvider.GetPodWatcher(ctx, targetNamespace) - err := startWatchLoop(ctx, watcher, podFilter, addedChan, modifiedChan, removedChan) // blocking + watcher, err := watcherCreator.NewWatcher(ctx, targetNamespace) + if err != nil { + errorChan <- fmt.Errorf("error in k8s watch: %v", err) + break + } + + err = startWatchLoop(ctx, watcher, filterer, eventChan) // blocking watcher.Stop() select { @@ -43,7 +51,7 @@ func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetName } if err != nil { - errorChan <- fmt.Errorf("error in k8 watch: %v", err) + errorChan <- fmt.Errorf("error in k8s watch: %v", err) break } else { if !watchRestartDebouncer.IsOn() { @@ -63,16 +71,14 @@ func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetName go func() { <-ctx.Done() wg.Wait() - close(addedChan) - close(modifiedChan) - close(removedChan) + close(eventChan) close(errorChan) }() - return addedChan, modifiedChan, removedChan, errorChan + return eventChan, errorChan } -func startWatchLoop(ctx context.Context, watcher watch.Interface, podFilter *regexp.Regexp, addedChan chan *corev1.Pod, modifiedChan chan *corev1.Pod, removedChan chan *corev1.Pod) error { +func startWatchLoop(ctx context.Context, watcher watch.Interface, filterer EventFilterer, eventChan chan<- *WatchEvent) error { resultChan := watcher.ResultChan() for { select { @@ -81,27 +87,19 @@ func startWatchLoop(ctx context.Context, watcher watch.Interface, podFilter *reg return nil } - if e.Type == watch.Error { - return apierrors.FromObject(e.Object) + wEvent := WatchEvent(e) + + if wEvent.Type == watch.Error { + return wEvent.ToError() } - pod, ok := e.Object.(*corev1.Pod) - if !ok { + if pass, err := filterer.Filter(&wEvent); err != nil { + return err + } else if !pass { continue } - if !podFilter.MatchString(pod.Name) { - continue - } - - switch e.Type { - case watch.Added: - addedChan <- pod - case watch.Modified: - modifiedChan <- pod - case watch.Deleted: - removedChan <- pod - } + eventChan <- &wEvent case <-ctx.Done(): return nil } diff --git a/shared/kubernetes/watchEvent.go b/shared/kubernetes/watchEvent.go new file mode 100644 index 000000000..81e1d0b09 --- /dev/null +++ b/shared/kubernetes/watchEvent.go @@ -0,0 +1,52 @@ +package kubernetes + +import ( + "fmt" + "reflect" + + corev1 "k8s.io/api/core/v1" + eventsv1 "k8s.io/api/events/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/watch" +) + +const ( + EventAdded watch.EventType = watch.Added + EventModified watch.EventType = watch.Modified + EventDeleted watch.EventType = watch.Deleted + EventBookmark watch.EventType = watch.Bookmark + EventError watch.EventType = watch.Error +) + +type InvalidObjectType struct { + RequestedType reflect.Type +} + +// Implements the error interface +func (iot *InvalidObjectType) Error() string { + return fmt.Sprintf("Cannot convert event to type %s", iot.RequestedType) +} + +type WatchEvent watch.Event + +func (we *WatchEvent) ToPod() (*corev1.Pod, error) { + pod, ok := we.Object.(*corev1.Pod) + if !ok { + return nil, &InvalidObjectType{RequestedType: reflect.TypeOf(pod)} + } + + return pod, nil +} + +func (we *WatchEvent) ToEvent() (*eventsv1.Event, error) { + event, ok := we.Object.(*eventsv1.Event) + if !ok { + return nil, &InvalidObjectType{RequestedType: reflect.TypeOf(event)} + } + + return event, nil +} + +func (we *WatchEvent) ToError() error { + return apierrors.FromObject(we.Object) +} diff --git a/shared/logger/logger.go b/shared/logger/logger.go index 305246550..52379a7a6 100644 --- a/shared/logger/logger.go +++ b/shared/logger/logger.go @@ -9,7 +9,7 @@ import ( var Log = logging.MustGetLogger("mizu") var format = logging.MustStringFormatter( - `%{time} %{level:.5s} â–¶ %{pid} %{shortfile} %{shortfunc} â–¶ %{message}`, + `[%{time:2006-01-02T15:04:05.000-0700}] %{level:-5s} â–¶ %{message} â–¶ [%{pid} %{shortfile} %{shortfunc}]`, ) func InitLogger(logPath string) { diff --git a/shared/models.go b/shared/models.go index 10b47883b..ef0cb2949 100644 --- a/shared/models.go +++ b/shared/models.go @@ -1,9 +1,10 @@ package shared import ( + "github.com/op/go-logging" + "github.com/up9inc/mizu/shared/logger" "github.com/up9inc/mizu/tap/api" "io/ioutil" - "log" "strings" "gopkg.in/yaml.v3" @@ -17,6 +18,9 @@ const ( WebSocketMessageTypeUpdateStatus WebSocketMessageType = "status" WebSocketMessageTypeAnalyzeStatus WebSocketMessageType = "analyzeStatus" WebsocketMessageTypeOutboundLink WebSocketMessageType = "outboundLink" + WebSocketMessageTypeToast WebSocketMessageType = "toast" + WebSocketMessageTypeQueryMetadata WebSocketMessageType = "queryMetadata" + WebSocketMessageTypeStartTime WebSocketMessageType = "startTime" ) type Resources struct { @@ -33,12 +37,13 @@ type MizuAgentConfig struct { TargetNamespaces []string `json:"targetNamespaces"` AgentImage string `json:"agentImage"` PullPolicy string `json:"pullPolicy"` - DumpLogs bool `json:"dumpLogs"` + LogLevel logging.Level `json:"logLevel"` IgnoredUserAgents []string `json:"ignoredUserAgents"` TapperResources Resources `json:"tapperResources"` MizuResourcesNamespace string `json:"mizuResourceNamespace"` MizuApiFilteringOptions api.TrafficFilteringOptions `json:"mizuApiFilteringOptions"` AgentDatabasePath string `json:"agentDatabasePath"` + Istio bool `json:"istio"` } type WebSocketMessageMetadata struct { @@ -59,17 +64,29 @@ type AnalyzeStatus struct { type WebSocketStatusMessage struct { *WebSocketMessageMetadata - TappingStatus TapStatus `json:"tappingStatus"` + TappingStatus []TappedPodStatus `json:"tappingStatus"` +} + +type TapperStatus struct { + TapperName string `json:"tapperName"` + NodeName string `json:"nodeName"` + Status string `json:"status"` +} + +type TappedPodStatus struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + IsTapped bool `json:"isTapped"` } type TapStatus struct { - Pods []PodInfo `json:"pods"` - TLSLinks []TLSLinkInfo `json:"tlsLinks"` + Pods []PodInfo `json:"pods"` } type PodInfo struct { Namespace string `json:"namespace"` Name string `json:"name"` + NodeName string `json:"nodeName"` } type TLSLinkInfo struct { @@ -86,12 +103,12 @@ type SyncEntriesConfig struct { UploadIntervalSec int `json:"interval"` } -func CreateWebSocketStatusMessage(tappingStatus TapStatus) WebSocketStatusMessage { +func CreateWebSocketStatusMessage(tappedPodsStatus []TappedPodStatus) WebSocketStatusMessage { return WebSocketStatusMessage{ WebSocketMessageMetadata: &WebSocketMessageMetadata{ MessageType: WebSocketMessageTypeUpdateStatus, }, - TappingStatus: tappingStatus, + TappingStatus: tappedPodsStatus, } } @@ -105,8 +122,9 @@ func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSoc } type HealthResponse struct { - TapStatus TapStatus `json:"tapStatus"` - TappersCount int `json:"tappersCount"` + TapStatus TapStatus `json:"tapStatus"` + TappersCount int `json:"tappersCount"` + TappersStatus []TapperStatus `json:"tappersStatus"` } type VersionResponse struct { @@ -137,14 +155,12 @@ func (r *RulePolicy) validateType() bool { permitedTypes := []string{"json", "header", "slo"} _, found := Find(permitedTypes, r.Type) if !found { - log.Printf("Error: %s. ", r.Name) - log.Printf("Only json, header and slo types are supported on rule definition. This rule will be ignored\n") + logger.Log.Errorf("Only json, header and slo types are supported on rule definition. This rule will be ignored. rule name: %s", r.Name) found = false } if strings.ToLower(r.Type) == "slo" { if r.ResponseTime <= 0 { - log.Printf("Error: %s. ", r.Name) - log.Printf("When type=slo, the field response-time should be specified and have a value >= 1\n\n") + logger.Log.Errorf("When rule type is slo, the field response-time should be specified and have a value >= 1. rule name: %s", r.Name) found = false } } diff --git a/tap/api/api.go b/tap/api/api.go index 5de34befe..3b1f9834a 100644 --- a/tap/api/api.go +++ b/tap/api/api.go @@ -18,7 +18,8 @@ import ( type Protocol struct { Name string `json:"name"` LongName string `json:"longName"` - Abbreviation string `json:"abbreviation"` + Abbreviation string `json:"abbr"` + Macro string `json:"macro"` Version string `json:"version"` BackgroundColor string `json:"backgroundColor"` ForegroundColor string `json:"foregroundColor"` @@ -28,6 +29,12 @@ type Protocol struct { Priority uint8 `json:"priority"` } +type TCP struct { + IP string `json:"ip"` + Port string `json:"port"` + Name string `json:"name"` +} + type Extension struct { Protocol *Protocol Path string @@ -74,6 +81,7 @@ type OutputChannelItem struct { Timestamp int64 ConnectionInfo *ConnectionInfo Pair *RequestResponsePair + Summary *BaseEntryDetails } type SuperTimer struct { @@ -89,9 +97,10 @@ type Dissector interface { Register(*Extension) Ping() Dissect(b *bufio.Reader, isClient bool, tcpID *TcpID, counterPair *CounterPair, superTimer *SuperTimer, superIdentifier *SuperIdentifier, emitter Emitter, options *TrafficFilteringOptions) error - Analyze(item *OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *MizuEntry + Analyze(item *OutputChannelItem, resolvedSource string, resolvedDestination string) *MizuEntry Summarize(entry *MizuEntry) *BaseEntryDetails - Represent(entry *MizuEntry) (protocol Protocol, object []byte, bodySize int64, err error) + Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) + Macros() map[string]string } type Emitting struct { @@ -109,39 +118,27 @@ func (e *Emitting) Emit(item *OutputChannelItem) { } type MizuEntry struct { - ID uint `gorm:"primarykey"` - CreatedAt time.Time - UpdatedAt time.Time - ProtocolName string `json:"protocolName" gorm:"column:protocolName"` - ProtocolLongName string `json:"protocolLongName" gorm:"column:protocolLongName"` - ProtocolAbbreviation string `json:"protocolAbbreviation" gorm:"column:protocolAbbreviation"` - ProtocolVersion string `json:"protocolVersion" gorm:"column:protocolVersion"` - ProtocolBackgroundColor string `json:"protocolBackgroundColor" gorm:"column:protocolBackgroundColor"` - ProtocolForegroundColor string `json:"protocolForegroundColor" gorm:"column:protocolForegroundColor"` - ProtocolFontSize int8 `json:"protocolFontSize" gorm:"column:protocolFontSize"` - ProtocolReferenceLink string `json:"protocolReferenceLink" gorm:"column:protocolReferenceLink"` - Entry string `json:"entry,omitempty" gorm:"column:entry"` - EntryId string `json:"entryId" gorm:"column:entryId"` - Url string `json:"url" gorm:"column:url"` - Method string `json:"method" gorm:"column:method"` - Status int `json:"status" gorm:"column:status"` - RequestSenderIp string `json:"requestSenderIp" gorm:"column:requestSenderIp"` - Service string `json:"service" gorm:"column:service"` - Timestamp int64 `json:"timestamp" gorm:"column:timestamp"` - ElapsedTime int64 `json:"elapsedTime" gorm:"column:elapsedTime"` - Path string `json:"path" gorm:"column:path"` - ResolvedSource string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"` - ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` - SourceIp string `json:"sourceIp,omitempty" gorm:"column:sourceIp"` - DestinationIp string `json:"destinationIp,omitempty" gorm:"column:destinationIp"` - SourcePort string `json:"sourcePort,omitempty" gorm:"column:sourcePort"` - DestinationPort string `json:"destinationPort,omitempty" gorm:"column:destinationPort"` - IsOutgoing bool `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"` - ContractStatus ContractStatus `json:"contractStatus,omitempty" gorm:"column:contractStatus"` - ContractRequestReason string `json:"contractRequestReason,omitempty" gorm:"column:contractRequestReason"` - ContractResponseReason string `json:"contractResponseReason,omitempty" gorm:"column:contractResponseReason"` - ContractContent string `json:"contractContent,omitempty" gorm:"column:contractContent"` - EstimatedSizeBytes int `json:"-" gorm:"column:estimatedSizeBytes"` + Id uint `json:"id"` + Protocol Protocol `json:"proto"` + Source *TCP `json:"src"` + Destination *TCP `json:"dst"` + Outgoing bool `json:"outgoing"` + Timestamp int64 `json:"timestamp"` + StartTime time.Time `json:"startTime"` + Request map[string]interface{} `json:"request"` + Response map[string]interface{} `json:"response"` + Base *BaseEntryDetails `json:"base"` + Summary string `json:"summary"` + Method string `json:"method"` + Status int `json:"status"` + ElapsedTime int64 `json:"elapsedTime"` + Path string `json:"path"` + IsOutgoing bool `json:"isOutgoing,omitempty"` + ContractStatus ContractStatus `json:"contractStatus,omitempty"` + ContractRequestReason string `json:"contractRequestReason,omitempty"` + ContractResponseReason string `json:"contractResponseReason,omitempty"` + ContractContent string `json:"contractContent,omitempty"` + HTTPPair string `json:"httpPair,omitempty"` } type MizuEntryWrapper struct { @@ -154,24 +151,20 @@ type MizuEntryWrapper struct { } type BaseEntryDetails struct { - Id string `json:"id,omitempty"` - Protocol Protocol `json:"protocol,omitempty"` - Url string `json:"url,omitempty"` - RequestSenderIp string `json:"requestSenderIp,omitempty"` - Service string `json:"service,omitempty"` - Path string `json:"path,omitempty"` - Summary string `json:"summary,omitempty"` - StatusCode int `json:"statusCode"` - Method string `json:"method,omitempty"` - Timestamp int64 `json:"timestamp,omitempty"` - SourceIp string `json:"sourceIp,omitempty"` - DestinationIp string `json:"destinationIp,omitempty"` - SourcePort string `json:"sourcePort,omitempty"` - DestinationPort string `json:"destinationPort,omitempty"` - IsOutgoing bool `json:"isOutgoing,omitempty"` - Latency int64 `json:"latency"` - Rules ApplicableRules `json:"rules,omitempty"` - ContractStatus ContractStatus `json:"contractStatus"` + Id uint `json:"id"` + Protocol Protocol `json:"protocol,omitempty"` + Url string `json:"url,omitempty"` + Path string `json:"path,omitempty"` + Summary string `json:"summary,omitempty"` + StatusCode int `json:"statusCode"` + Method string `json:"method,omitempty"` + Timestamp int64 `json:"timestamp,omitempty"` + Source *TCP `json:"src"` + Destination *TCP `json:"dst"` + IsOutgoing bool `json:"isOutgoing,omitempty"` + Latency int64 `json:"latency"` + Rules ApplicableRules `json:"rules,omitempty"` + ContractStatus ContractStatus `json:"contractStatus"` } type ApplicableRules struct { @@ -194,29 +187,15 @@ type DataUnmarshaler interface { } func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error { - bed.Protocol = Protocol{ - Name: entry.ProtocolName, - LongName: entry.ProtocolLongName, - Abbreviation: entry.ProtocolAbbreviation, - Version: entry.ProtocolVersion, - BackgroundColor: entry.ProtocolBackgroundColor, - ForegroundColor: entry.ProtocolForegroundColor, - FontSize: entry.ProtocolFontSize, - ReferenceLink: entry.ProtocolReferenceLink, - } - bed.Id = entry.EntryId - bed.Url = entry.Url - bed.RequestSenderIp = entry.RequestSenderIp - bed.Service = entry.Service + bed.Protocol = entry.Protocol + bed.Id = entry.Id bed.Path = entry.Path - bed.Summary = entry.Path + bed.Summary = entry.Summary bed.StatusCode = entry.Status bed.Method = entry.Method bed.Timestamp = entry.Timestamp - bed.SourceIp = entry.SourceIp - bed.DestinationIp = entry.DestinationIp - bed.SourcePort = entry.SourcePort - bed.DestinationPort = entry.DestinationPort + bed.Source = entry.Source + bed.Destination = entry.Destination bed.IsOutgoing = entry.IsOutgoing bed.Latency = entry.ElapsedTime bed.ContractStatus = entry.ContractStatus @@ -228,6 +207,21 @@ const ( BODY string = "body" ) +type SectionData struct { + Type string `json:"type"` + Title string `json:"title"` + Data string `json:"data"` + Encoding string `json:"encoding,omitempty"` + MimeType string `json:"mimeType,omitempty"` + Selector string `json:"selector,omitempty"` +} + +type TableData struct { + Name string `json:"name"` + Value interface{} `json:"value"` + Selector string `json:"selector"` +} + const ( TypeHttpRequest = iota TypeHttpResponse @@ -259,7 +253,6 @@ func (h HTTPPayload) MarshalJSON() ([]byte, error) { } return json.Marshal(&HTTPWrapper{ Method: harRequest.Method, - Url: "", Details: harRequest, RawRequest: &HTTPRequestWrapper{Request: h.Data.(*http.Request)}, }) @@ -275,7 +268,7 @@ func (h HTTPPayload) MarshalJSON() ([]byte, error) { RawResponse: &HTTPResponseWrapper{Response: h.Data.(*http.Response)}, }) default: - panic(fmt.Sprintf("HTTP payload cannot be marshaled: %s\n", h.Type)) + panic(fmt.Sprintf("HTTP payload cannot be marshaled: %s", h.Type)) } } diff --git a/tap/diagnose/diagnose.go b/tap/diagnose/diagnose.go index 2c64a3b35..f40fd5e27 100644 --- a/tap/diagnose/diagnose.go +++ b/tap/diagnose/diagnose.go @@ -39,7 +39,7 @@ func StartMemoryProfiler(envDumpPath string, envTimeInterval string) { filename := fmt.Sprintf("%s/%s__mem.prof", dumpPath, t.Format("15_04_05")) - logger.Log.Infof("Writing memory profile to %s\n", filename) + logger.Log.Infof("Writing memory profile to %s", filename) f, err := os.Create(filename) if err != nil { diff --git a/tap/extensions/amqp/helpers.go b/tap/extensions/amqp/helpers.go index f7a1842a7..2c3a8b571 100644 --- a/tap/extensions/amqp/helpers.go +++ b/tap/extensions/amqp/helpers.go @@ -131,97 +131,109 @@ func representProperties(properties map[string]interface{}, rep []interface{}) ( userId := "" appId := "" - if properties["ContentType"] != nil { - contentType = properties["ContentType"].(string) + if properties["contentType"] != nil { + contentType = properties["contentType"].(string) } - if properties["ContentEncoding"] != nil { - contentEncoding = properties["ContentEncoding"].(string) + if properties["contentEncoding"] != nil { + contentEncoding = properties["contentEncoding"].(string) } - if properties["Delivery Mode"] != nil { - deliveryMode = fmt.Sprintf("%g", properties["DeliveryMode"].(float64)) + if properties["deliveryMode"] != nil { + deliveryMode = fmt.Sprintf("%g", properties["deliveryMode"].(float64)) } - if properties["Priority"] != nil { - priority = fmt.Sprintf("%g", properties["Priority"].(float64)) + if properties["priority"] != nil { + priority = fmt.Sprintf("%g", properties["priority"].(float64)) } - if properties["CorrelationId"] != nil { - correlationId = properties["CorrelationId"].(string) + if properties["correlationId"] != nil { + correlationId = properties["correlationId"].(string) } - if properties["ReplyTo"] != nil { - replyTo = properties["ReplyTo"].(string) + if properties["replyTo"] != nil { + replyTo = properties["replyTo"].(string) } - if properties["Expiration"] != nil { - expiration = properties["Expiration"].(string) + if properties["expiration"] != nil { + expiration = properties["expiration"].(string) } - if properties["MessageId"] != nil { - messageId = properties["MessageId"].(string) + if properties["messageId"] != nil { + messageId = properties["messageId"].(string) } - if properties["Timestamp"] != nil { - timestamp = properties["Timestamp"].(string) + if properties["timestamp"] != nil { + timestamp = properties["timestamp"].(string) } - if properties["Type"] != nil { - _type = properties["Type"].(string) + if properties["type"] != nil { + _type = properties["type"].(string) } - if properties["UserId"] != nil { - userId = properties["UserId"].(string) + if properties["userId"] != nil { + userId = properties["userId"].(string) } - if properties["AppId"] != nil { - appId = properties["AppId"].(string) + if properties["appId"] != nil { + appId = properties["appId"].(string) } - props, _ := json.Marshal([]map[string]string{ + props, _ := json.Marshal([]api.TableData{ { - "name": "Content Type", - "value": contentType, + Name: "Content Type", + Value: contentType, + Selector: `request.properties.contentType`, }, { - "name": "Content Encoding", - "value": contentEncoding, + Name: "Content Encoding", + Value: contentEncoding, + Selector: `request.properties.contentEncoding`, }, { - "name": "Delivery Mode", - "value": deliveryMode, + Name: "Delivery Mode", + Value: deliveryMode, + Selector: `request.properties.deliveryMode`, }, { - "name": "Priority", - "value": priority, + Name: "Priority", + Value: priority, + Selector: `request.properties.priority`, }, { - "name": "Correlation ID", - "value": correlationId, + Name: "Correlation ID", + Value: correlationId, + Selector: `request.properties.correlationId`, }, { - "name": "Reply To", - "value": replyTo, + Name: "Reply To", + Value: replyTo, + Selector: `request.properties.replyTo`, }, { - "name": "Expiration", - "value": expiration, + Name: "Expiration", + Value: expiration, + Selector: `request.properties.expiration`, }, { - "name": "Message ID", - "value": messageId, + Name: "Message ID", + Value: messageId, + Selector: `request.properties.messageId`, }, { - "name": "Timestamp", - "value": timestamp, + Name: "Timestamp", + Value: timestamp, + Selector: `request.properties.timestamp`, }, { - "name": "Type", - "value": _type, + Name: "Type", + Value: _type, + Selector: `request.properties.type`, }, { - "name": "User ID", - "value": userId, + Name: "User ID", + Value: userId, + Selector: `request.properties.userId`, }, { - "name": "App ID", - "value": appId, + Name: "App ID", + Value: appId, + Selector: `request.properties.appId`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Properties", - "data": string(props), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Properties", + Data: string(props), }) return rep, contentType, contentEncoding @@ -230,56 +242,62 @@ func representProperties(properties map[string]interface{}, rep []interface{}) ( func representBasicPublish(event map[string]interface{}) []interface{} { rep := make([]interface{}, 0) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Exchange", - "value": event["Exchange"].(string), + Name: "Exchange", + Value: event["exchange"].(string), + Selector: `request.exchange`, }, { - "name": "Routing Key", - "value": event["RoutingKey"].(string), + Name: "Routing Key", + Value: event["routingKey"].(string), + Selector: `request.routingKey`, }, { - "name": "Mandatory", - "value": strconv.FormatBool(event["Mandatory"].(bool)), + Name: "Mandatory", + Value: strconv.FormatBool(event["mandatory"].(bool)), + Selector: `request.mandatory`, }, { - "name": "Immediate", - "value": strconv.FormatBool(event["Immediate"].(bool)), + Name: "Immediate", + Value: strconv.FormatBool(event["immediate"].(bool)), + Selector: `request.immediate`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - properties := event["Properties"].(map[string]interface{}) + properties := event["properties"].(map[string]interface{}) rep, contentType, _ := representProperties(properties, rep) - if properties["Headers"] != nil { - headers := make([]map[string]string, 0) - for name, value := range properties["Headers"].(map[string]interface{}) { - headers = append(headers, map[string]string{ - "name": name, - "value": value.(string), + if properties["headers"] != nil { + headers := make([]api.TableData, 0) + for name, value := range properties["headers"].(map[string]interface{}) { + headers = append(headers, api.TableData{ + Name: name, + Value: value.(string), + Selector: fmt.Sprintf(`request.properties.headers["%s"]`, name), }) } headersMarshaled, _ := json.Marshal(headers) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Headers", - "data": string(headersMarshaled), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Headers", + Data: string(headersMarshaled), }) } - if event["Body"] != nil { - rep = append(rep, map[string]string{ - "type": api.BODY, - "title": "Body", - "encoding": "base64", - "mime_type": contentType, - "data": event["Body"].(string), + if event["body"] != nil { + rep = append(rep, api.SectionData{ + Type: api.BODY, + Title: "Body", + Encoding: "base64", + MimeType: contentType, + Data: event["body"].(string), + Selector: `request.body`, }) } @@ -293,70 +311,77 @@ func representBasicDeliver(event map[string]interface{}) []interface{} { deliveryTag := "" redelivered := "" - if event["ConsumerTag"] != nil { - consumerTag = event["ConsumerTag"].(string) + if event["consumerTag"] != nil { + consumerTag = event["consumerTag"].(string) } - if event["DeliveryTag"] != nil { - deliveryTag = fmt.Sprintf("%g", event["DeliveryTag"].(float64)) + if event["deliveryTag"] != nil { + deliveryTag = fmt.Sprintf("%g", event["deliveryTag"].(float64)) } - if event["Redelivered"] != nil { - redelivered = strconv.FormatBool(event["Redelivered"].(bool)) + if event["redelivered"] != nil { + redelivered = strconv.FormatBool(event["redelivered"].(bool)) } - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Consumer Tag", - "value": consumerTag, + Name: "Consumer Tag", + Value: consumerTag, + Selector: `request.consumerTag`, }, { - "name": "Delivery Tag", - "value": deliveryTag, + Name: "Delivery Tag", + Value: deliveryTag, + Selector: `request.deliveryTag`, }, { - "name": "Redelivered", - "value": redelivered, + Name: "Redelivered", + Value: redelivered, + Selector: `request.redelivered`, }, { - "name": "Exchange", - "value": event["Exchange"].(string), + Name: "Exchange", + Value: event["exchange"].(string), + Selector: `request.exchange`, }, { - "name": "Routing Key", - "value": event["RoutingKey"].(string), + Name: "Routing Key", + Value: event["routingKey"].(string), + Selector: `request.routingKey`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - properties := event["Properties"].(map[string]interface{}) + properties := event["properties"].(map[string]interface{}) rep, contentType, _ := representProperties(properties, rep) - if properties["Headers"] != nil { - headers := make([]map[string]string, 0) - for name, value := range properties["Headers"].(map[string]interface{}) { - headers = append(headers, map[string]string{ - "name": name, - "value": value.(string), + if properties["headers"] != nil { + headers := make([]api.TableData, 0) + for name, value := range properties["headers"].(map[string]interface{}) { + headers = append(headers, api.TableData{ + Name: name, + Value: value.(string), + Selector: fmt.Sprintf(`request.properties.headers["%s"]`, name), }) } headersMarshaled, _ := json.Marshal(headers) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Headers", - "data": string(headersMarshaled), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Headers", + Data: string(headersMarshaled), }) } - if event["Body"] != nil { - rep = append(rep, map[string]string{ - "type": api.BODY, - "title": "Body", - "encoding": "base64", - "mime_type": contentType, - "data": event["Body"].(string), + if event["body"] != nil { + rep = append(rep, api.SectionData{ + Type: api.BODY, + Title: "Body", + Encoding: "base64", + MimeType: contentType, + Data: event["body"].(string), + Selector: `request.body`, }) } @@ -366,51 +391,58 @@ func representBasicDeliver(event map[string]interface{}) []interface{} { func representQueueDeclare(event map[string]interface{}) []interface{} { rep := make([]interface{}, 0) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Queue", - "value": event["Queue"].(string), + Name: "Queue", + Value: event["queue"].(string), + Selector: `request.queue`, }, { - "name": "Passive", - "value": strconv.FormatBool(event["Passive"].(bool)), + Name: "Passive", + Value: strconv.FormatBool(event["passive"].(bool)), + Selector: `request.queue`, }, { - "name": "Durable", - "value": strconv.FormatBool(event["Durable"].(bool)), + Name: "Durable", + Value: strconv.FormatBool(event["durable"].(bool)), + Selector: `request.durable`, }, { - "name": "Exclusive", - "value": strconv.FormatBool(event["Exclusive"].(bool)), + Name: "Exclusive", + Value: strconv.FormatBool(event["exclusive"].(bool)), + Selector: `request.exclusive`, }, { - "name": "Auto Delete", - "value": strconv.FormatBool(event["AutoDelete"].(bool)), + Name: "Auto Delete", + Value: strconv.FormatBool(event["autoDelete"].(bool)), + Selector: `request.autoDelete`, }, { - "name": "NoWait", - "value": strconv.FormatBool(event["NoWait"].(bool)), + Name: "NoWait", + Value: strconv.FormatBool(event["noWait"].(bool)), + Selector: `request.noWait`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - if event["Arguments"] != nil { - headers := make([]map[string]string, 0) - for name, value := range event["Arguments"].(map[string]interface{}) { - headers = append(headers, map[string]string{ - "name": name, - "value": value.(string), + if event["arguments"] != nil { + headers := make([]api.TableData, 0) + for name, value := range event["arguments"].(map[string]interface{}) { + headers = append(headers, api.TableData{ + Name: name, + Value: value.(string), + Selector: fmt.Sprintf(`request.arguments["%s"]`, name), }) } headersMarshaled, _ := json.Marshal(headers) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Arguments", - "data": string(headersMarshaled), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Arguments", + Data: string(headersMarshaled), }) } @@ -420,55 +452,63 @@ func representQueueDeclare(event map[string]interface{}) []interface{} { func representExchangeDeclare(event map[string]interface{}) []interface{} { rep := make([]interface{}, 0) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Exchange", - "value": event["Exchange"].(string), + Name: "Exchange", + Value: event["exchange"].(string), + Selector: `request.exchange`, }, { - "name": "Type", - "value": event["Type"].(string), + Name: "Type", + Value: event["type"].(string), + Selector: `request.type`, }, { - "name": "Passive", - "value": strconv.FormatBool(event["Passive"].(bool)), + Name: "Passive", + Value: strconv.FormatBool(event["passive"].(bool)), + Selector: `request.passive`, }, { - "name": "Durable", - "value": strconv.FormatBool(event["Durable"].(bool)), + Name: "Durable", + Value: strconv.FormatBool(event["durable"].(bool)), + Selector: `request.durable`, }, { - "name": "Auto Delete", - "value": strconv.FormatBool(event["AutoDelete"].(bool)), + Name: "Auto Delete", + Value: strconv.FormatBool(event["autoDelete"].(bool)), + Selector: `request.autoDelete`, }, { - "name": "Internal", - "value": strconv.FormatBool(event["Internal"].(bool)), + Name: "Internal", + Value: strconv.FormatBool(event["internal"].(bool)), + Selector: `request.internal`, }, { - "name": "NoWait", - "value": strconv.FormatBool(event["NoWait"].(bool)), + Name: "NoWait", + Value: strconv.FormatBool(event["noWait"].(bool)), + Selector: `request.noWait`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - if event["Arguments"] != nil { - headers := make([]map[string]string, 0) - for name, value := range event["Arguments"].(map[string]interface{}) { - headers = append(headers, map[string]string{ - "name": name, - "value": value.(string), + if event["arguments"] != nil { + headers := make([]api.TableData, 0) + for name, value := range event["arguments"].(map[string]interface{}) { + headers = append(headers, api.TableData{ + Name: name, + Value: value.(string), + Selector: fmt.Sprintf(`request.arguments["%s"]`, name), }) } headersMarshaled, _ := json.Marshal(headers) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Arguments", - "data": string(headersMarshaled), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Arguments", + Data: string(headersMarshaled), }) } @@ -478,33 +518,37 @@ func representExchangeDeclare(event map[string]interface{}) []interface{} { func representConnectionStart(event map[string]interface{}) []interface{} { rep := make([]interface{}, 0) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Version Major", - "value": fmt.Sprintf("%g", event["VersionMajor"].(float64)), + Name: "Version Major", + Value: fmt.Sprintf("%g", event["versionMajor"].(float64)), + Selector: `request.versionMajor`, }, { - "name": "Version Minor", - "value": fmt.Sprintf("%g", event["VersionMinor"].(float64)), + Name: "Version Minor", + Value: fmt.Sprintf("%g", event["versionMinor"].(float64)), + Selector: `request.versionMinor`, }, { - "name": "Mechanisms", - "value": event["Mechanisms"].(string), + Name: "Mechanisms", + Value: event["mechanisms"].(string), + Selector: `request.mechanisms`, }, { - "name": "Locales", - "value": event["Locales"].(string), + Name: "Locales", + Value: event["locales"].(string), + Selector: `request.locales`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - if event["ServerProperties"] != nil { - headers := make([]map[string]string, 0) - for name, value := range event["ServerProperties"].(map[string]interface{}) { + if event["serverProperties"] != nil { + headers := make([]api.TableData, 0) + for name, value := range event["serverProperties"].(map[string]interface{}) { var outcome string switch value.(type) { case string: @@ -517,16 +561,17 @@ func representConnectionStart(event map[string]interface{}) []interface{} { default: panic("Unknown data type for the server property!") } - headers = append(headers, map[string]string{ - "name": name, - "value": outcome, + headers = append(headers, api.TableData{ + Name: name, + Value: outcome, + Selector: fmt.Sprintf(`request.serverProperties["%s"]`, name), }) } headersMarshaled, _ := json.Marshal(headers) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Server Properties", - "data": string(headersMarshaled), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Server Properties", + Data: string(headersMarshaled), }) } @@ -534,30 +579,40 @@ func representConnectionStart(event map[string]interface{}) []interface{} { } func representConnectionClose(event map[string]interface{}) []interface{} { + replyCode := "" + + if event["replyCode"] != nil { + replyCode = fmt.Sprintf("%g", event["replyCode"].(float64)) + } + rep := make([]interface{}, 0) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Reply Code", - "value": fmt.Sprintf("%g", event["ReplyCode"].(float64)), + Name: "Reply Code", + Value: replyCode, + Selector: `request.replyCode`, }, { - "name": "Reply Text", - "value": event["ReplyText"].(string), + Name: "Reply Text", + Value: event["replyText"].(string), + Selector: `request.replyText`, }, { - "name": "Class ID", - "value": fmt.Sprintf("%g", event["ClassId"].(float64)), + Name: "Class ID", + Value: fmt.Sprintf("%g", event["classId"].(float64)), + Selector: `request.classId`, }, { - "name": "Method ID", - "value": fmt.Sprintf("%g", event["MethodId"].(float64)), + Name: "Method ID", + Value: fmt.Sprintf("%g", event["methodId"].(float64)), + Selector: `request.methodId`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) return rep @@ -566,43 +621,48 @@ func representConnectionClose(event map[string]interface{}) []interface{} { func representQueueBind(event map[string]interface{}) []interface{} { rep := make([]interface{}, 0) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Queue", - "value": event["Queue"].(string), + Name: "Queue", + Value: event["queue"].(string), + Selector: `request.queue`, }, { - "name": "Exchange", - "value": event["Exchange"].(string), + Name: "Exchange", + Value: event["exchange"].(string), + Selector: `request.exchange`, }, { - "name": "RoutingKey", - "value": event["RoutingKey"].(string), + Name: "RoutingKey", + Value: event["routingKey"].(string), + Selector: `request.routingKey`, }, { - "name": "NoWait", - "value": strconv.FormatBool(event["NoWait"].(bool)), + Name: "NoWait", + Value: strconv.FormatBool(event["noWait"].(bool)), + Selector: `request.noWait`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - if event["Arguments"] != nil { - headers := make([]map[string]string, 0) - for name, value := range event["Arguments"].(map[string]interface{}) { - headers = append(headers, map[string]string{ - "name": name, - "value": value.(string), + if event["arguments"] != nil { + headers := make([]api.TableData, 0) + for name, value := range event["arguments"].(map[string]interface{}) { + headers = append(headers, api.TableData{ + Name: name, + Value: value.(string), + Selector: fmt.Sprintf(`request.arguments["%s"]`, name), }) } headersMarshaled, _ := json.Marshal(headers) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Arguments", - "data": string(headersMarshaled), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Arguments", + Data: string(headersMarshaled), }) } @@ -612,51 +672,58 @@ func representQueueBind(event map[string]interface{}) []interface{} { func representBasicConsume(event map[string]interface{}) []interface{} { rep := make([]interface{}, 0) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Queue", - "value": event["Queue"].(string), + Name: "Queue", + Value: event["queue"].(string), + Selector: `request.queue`, }, { - "name": "Consumer Tag", - "value": event["ConsumerTag"].(string), + Name: "Consumer Tag", + Value: event["consumerTag"].(string), + Selector: `request.consumerTag`, }, { - "name": "No Local", - "value": strconv.FormatBool(event["NoLocal"].(bool)), + Name: "No Local", + Value: strconv.FormatBool(event["noLocal"].(bool)), + Selector: `request.noLocal`, }, { - "name": "No Ack", - "value": strconv.FormatBool(event["NoAck"].(bool)), + Name: "No Ack", + Value: strconv.FormatBool(event["noAck"].(bool)), + Selector: `request.noAck`, }, { - "name": "Exclusive", - "value": strconv.FormatBool(event["Exclusive"].(bool)), + Name: "Exclusive", + Value: strconv.FormatBool(event["exclusive"].(bool)), + Selector: `request.exclusive`, }, { - "name": "NoWait", - "value": strconv.FormatBool(event["NoWait"].(bool)), + Name: "NoWait", + Value: strconv.FormatBool(event["noWait"].(bool)), + Selector: `request.noWait`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - if event["Arguments"] != nil { - headers := make([]map[string]string, 0) - for name, value := range event["Arguments"].(map[string]interface{}) { - headers = append(headers, map[string]string{ - "name": name, - "value": value.(string), + if event["arguments"] != nil { + headers := make([]api.TableData, 0) + for name, value := range event["arguments"].(map[string]interface{}) { + headers = append(headers, api.TableData{ + Name: name, + Value: value.(string), + Selector: fmt.Sprintf(`request.arguments["%s"]`, name), }) } headersMarshaled, _ := json.Marshal(headers) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Arguments", - "data": string(headersMarshaled), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Arguments", + Data: string(headersMarshaled), }) } diff --git a/tap/extensions/amqp/main.go b/tap/extensions/amqp/main.go index 133e8997a..01bd2fac6 100644 --- a/tap/extensions/amqp/main.go +++ b/tap/extensions/amqp/main.go @@ -16,6 +16,7 @@ var protocol api.Protocol = api.Protocol{ Name: "amqp", LongName: "Advanced Message Queuing Protocol 0-9-1", Abbreviation: "AMQP", + Macro: "amqp", Version: "0-9-1", BackgroundColor: "#ff6600", ForegroundColor: "#ffffff", @@ -36,7 +37,7 @@ func (d dissecting) Register(extension *api.Extension) { } func (d dissecting) Ping() { - log.Printf("pong %s\n", protocol.Name) + log.Printf("pong %s", protocol.Name) } const amqpRequest string = "amqp_request" @@ -217,102 +218,86 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co } default: - // log.Printf("unexpected frame: %+v\n", f) + // log.Printf("unexpected frame: %+v", f) } } } -func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry { +func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry { request := item.Pair.Request.Payload.(map[string]interface{}) reqDetails := request["details"].(map[string]interface{}) - service := "amqp" - if resolvedDestination != "" { - service = resolvedDestination - } else if resolvedSource != "" { - service = resolvedSource - } summary := "" switch request["method"] { case basicMethodMap[40]: - summary = reqDetails["Exchange"].(string) + summary = reqDetails["exchange"].(string) break case basicMethodMap[60]: - summary = reqDetails["Exchange"].(string) + summary = reqDetails["exchange"].(string) break case exchangeMethodMap[10]: - summary = reqDetails["Exchange"].(string) + summary = reqDetails["exchange"].(string) break case queueMethodMap[10]: - summary = reqDetails["Queue"].(string) + summary = reqDetails["queue"].(string) break case connectionMethodMap[10]: summary = fmt.Sprintf( "%s.%s", - strconv.Itoa(int(reqDetails["VersionMajor"].(float64))), - strconv.Itoa(int(reqDetails["VersionMinor"].(float64))), + strconv.Itoa(int(reqDetails["versionMajor"].(float64))), + strconv.Itoa(int(reqDetails["versionMinor"].(float64))), ) break case connectionMethodMap[50]: - summary = reqDetails["ReplyText"].(string) + summary = reqDetails["replyText"].(string) break case queueMethodMap[20]: - summary = reqDetails["Queue"].(string) + summary = reqDetails["queue"].(string) break case basicMethodMap[20]: - summary = reqDetails["Queue"].(string) + summary = reqDetails["queue"].(string) break } request["url"] = summary - entryBytes, _ := json.Marshal(item.Pair) + reqDetails["method"] = request["method"] return &api.MizuEntry{ - ProtocolName: protocol.Name, - ProtocolLongName: protocol.LongName, - ProtocolAbbreviation: protocol.Abbreviation, - ProtocolVersion: protocol.Version, - ProtocolBackgroundColor: protocol.BackgroundColor, - ProtocolForegroundColor: protocol.ForegroundColor, - ProtocolFontSize: protocol.FontSize, - ProtocolReferenceLink: protocol.ReferenceLink, - EntryId: entryId, - Entry: string(entryBytes), - Url: fmt.Sprintf("%s%s", service, summary), - Method: request["method"].(string), - Status: 0, - RequestSenderIp: item.ConnectionInfo.ClientIP, - Service: service, - Timestamp: item.Timestamp, - ElapsedTime: 0, - Path: summary, - ResolvedSource: resolvedSource, - ResolvedDestination: resolvedDestination, - SourceIp: item.ConnectionInfo.ClientIP, - DestinationIp: item.ConnectionInfo.ServerIP, - SourcePort: item.ConnectionInfo.ClientPort, - DestinationPort: item.ConnectionInfo.ServerPort, - IsOutgoing: item.ConnectionInfo.IsOutgoing, + Protocol: protocol, + Source: &api.TCP{ + Name: resolvedSource, + IP: item.ConnectionInfo.ClientIP, + Port: item.ConnectionInfo.ClientPort, + }, + Destination: &api.TCP{ + Name: resolvedDestination, + IP: item.ConnectionInfo.ServerIP, + Port: item.ConnectionInfo.ServerPort, + }, + Outgoing: item.ConnectionInfo.IsOutgoing, + Request: reqDetails, + Method: request["method"].(string), + Status: 0, + Timestamp: item.Timestamp, + StartTime: item.Pair.Request.CaptureTime, + ElapsedTime: 0, + Summary: summary, + IsOutgoing: item.ConnectionInfo.IsOutgoing, } } func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { return &api.BaseEntryDetails{ - Id: entry.EntryId, - Protocol: protocol, - Url: entry.Url, - RequestSenderIp: entry.RequestSenderIp, - Service: entry.Service, - Summary: entry.Path, - StatusCode: entry.Status, - Method: entry.Method, - Timestamp: entry.Timestamp, - SourceIp: entry.SourceIp, - DestinationIp: entry.DestinationIp, - SourcePort: entry.SourcePort, - DestinationPort: entry.DestinationPort, - IsOutgoing: entry.IsOutgoing, - Latency: entry.ElapsedTime, + Id: entry.Id, + Protocol: protocol, + Summary: entry.Summary, + StatusCode: entry.Status, + Method: entry.Method, + Timestamp: entry.Timestamp, + Source: entry.Source, + Destination: entry.Destination, + IsOutgoing: entry.IsOutgoing, + Latency: entry.ElapsedTime, Rules: api.ApplicableRules{ Latency: 0, Status: false, @@ -320,39 +305,34 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { } } -func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) { - p = protocol +func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) { bodySize = 0 - var root map[string]interface{} - json.Unmarshal([]byte(entry.Entry), &root) representation := make(map[string]interface{}, 0) - request := root["request"].(map[string]interface{})["payload"].(map[string]interface{}) var repRequest []interface{} - details := request["details"].(map[string]interface{}) switch request["method"].(string) { case basicMethodMap[40]: - repRequest = representBasicPublish(details) + repRequest = representBasicPublish(request) break case basicMethodMap[60]: - repRequest = representBasicDeliver(details) + repRequest = representBasicDeliver(request) break case queueMethodMap[10]: - repRequest = representQueueDeclare(details) + repRequest = representQueueDeclare(request) break case exchangeMethodMap[10]: - repRequest = representExchangeDeclare(details) + repRequest = representExchangeDeclare(request) break case connectionMethodMap[10]: - repRequest = representConnectionStart(details) + repRequest = representConnectionStart(request) break case connectionMethodMap[50]: - repRequest = representConnectionClose(details) + repRequest = representConnectionClose(request) break case queueMethodMap[20]: - repRequest = representQueueBind(details) + repRequest = representQueueBind(request) break case basicMethodMap[20]: - repRequest = representBasicConsume(details) + repRequest = representBasicConsume(request) break } representation["request"] = repRequest @@ -360,4 +340,10 @@ func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []by return } +func (d dissecting) Macros() map[string]string { + return map[string]string{ + `amqp`: fmt.Sprintf(`proto.name == "%s"`, protocol.Name), + } +} + var Dissector dissecting diff --git a/tap/extensions/amqp/spec091.go b/tap/extensions/amqp/spec091.go index 3dd3dcbd3..0af77a26c 100644 --- a/tap/extensions/amqp/spec091.go +++ b/tap/extensions/amqp/spec091.go @@ -71,11 +71,11 @@ func isSoftExceptionCode(code int) bool { } type ConnectionStart struct { - VersionMajor byte - VersionMinor byte - ServerProperties Table - Mechanisms string - Locales string + VersionMajor byte `json:"versionMajor"` + VersionMinor byte `json:"versionMinor"` + ServerProperties Table `json:"serverProperties"` + Mechanisms string `json:"mechanisms"` + Locales string `json:"locales"` } func (msg *ConnectionStart) id() (uint16, uint16) { @@ -429,10 +429,10 @@ func (msg *connectionOpenOk) read(r io.Reader) (err error) { } type ConnectionClose struct { - ReplyCode uint16 - ReplyText string - ClassId uint16 - MethodId uint16 + ReplyCode uint16 `json:"relyCode"` + ReplyText string `json:"replyText"` + ClassId uint16 `json:"classId"` + MethodId uint16 `json:"methodId"` } func (msg *ConnectionClose) id() (uint16, uint16) { @@ -767,14 +767,14 @@ func (msg *channelCloseOk) read(r io.Reader) (err error) { type ExchangeDeclare struct { reserved1 uint16 - Exchange string - Type string - Passive bool - Durable bool - AutoDelete bool - Internal bool - NoWait bool - Arguments Table + Exchange string `json:"exchange"` + Type string `json:"type"` + Passive bool `json:"passive"` + Durable bool `json:"durable"` + AutoDelete bool `json:"autoDelete"` + Internal bool `json:"internal"` + NoWait bool `json:"noWait"` + Arguments Table `json:"arguments"` } func (msg *ExchangeDeclare) id() (uint16, uint16) { @@ -1163,13 +1163,13 @@ func (msg *exchangeUnbindOk) read(r io.Reader) (err error) { type QueueDeclare struct { reserved1 uint16 - Queue string - Passive bool - Durable bool - Exclusive bool - AutoDelete bool - NoWait bool - Arguments Table + Queue string `json:"queue"` + Passive bool `json:"passive"` + Durable bool `json:"durable"` + Exclusive bool `json:"exclusive"` + AutoDelete bool `json:"autoDelete"` + NoWait bool `json:"noWait"` + Arguments Table `json:"arguments"` } func (msg *QueueDeclare) id() (uint16, uint16) { @@ -1297,11 +1297,11 @@ func (msg *QueueDeclareOk) read(r io.Reader) (err error) { type QueueBind struct { reserved1 uint16 - Queue string - Exchange string - RoutingKey string - NoWait bool - Arguments Table + Queue string `json:"queue"` + Exchange string `json:"exchange"` + RoutingKey string `json:"routingKey"` + NoWait bool `json:"noWait"` + Arguments Table `json:"arguments"` } func (msg *QueueBind) id() (uint16, uint16) { @@ -1737,13 +1737,13 @@ func (msg *basicQosOk) read(r io.Reader) (err error) { type BasicConsume struct { reserved1 uint16 - Queue string - ConsumerTag string - NoLocal bool - NoAck bool - Exclusive bool - NoWait bool - Arguments Table + Queue string `json:"queue"` + ConsumerTag string `json:"consumerTag"` + NoLocal bool `json:"noLocal"` + NoAck bool `json:"noAck"` + Exclusive bool `json:"exclusive"` + NoWait bool `json:"noWait"` + Arguments Table `json:"arguments"` } func (msg *BasicConsume) id() (uint16, uint16) { @@ -1932,12 +1932,12 @@ func (msg *basicCancelOk) read(r io.Reader) (err error) { type BasicPublish struct { reserved1 uint16 - Exchange string - RoutingKey string - Mandatory bool - Immediate bool - Properties Properties - Body []byte + Exchange string `json:"exchange"` + RoutingKey string `json:"routingKey"` + Mandatory bool `json:"mandatory"` + Immediate bool `json:"immediate"` + Properties Properties `json:"properties"` + Body []byte `json:"body"` } func (msg *BasicPublish) id() (uint16, uint16) { @@ -2072,13 +2072,13 @@ func (msg *basicReturn) read(r io.Reader) (err error) { } type BasicDeliver struct { - ConsumerTag string - DeliveryTag uint64 - Redelivered bool - Exchange string - RoutingKey string - Properties Properties - Body []byte + ConsumerTag string `json:"consumerTag"` + DeliveryTag uint64 `json:"deliveryTag"` + Redelivered bool `json:"redelivered"` + Exchange string `json:"exchange"` + RoutingKey string `json:"routingKey"` + Properties Properties `json:"properties"` + Body []byte `json:"body"` } func (msg *BasicDeliver) id() (uint16, uint16) { diff --git a/tap/extensions/amqp/types.go b/tap/extensions/amqp/types.go index ea57556fa..1adee51c6 100644 --- a/tap/extensions/amqp/types.go +++ b/tap/extensions/amqp/types.go @@ -93,19 +93,19 @@ func (e Error) Error() string { // Used by header frames to capture routing and header information type Properties struct { - ContentType string // MIME content type - ContentEncoding string // MIME content encoding - Headers Table // Application or header exchange table - DeliveryMode uint8 // queue implementation use - Transient (1) or Persistent (2) - Priority uint8 // queue implementation use - 0 to 9 - CorrelationId string // application use - correlation identifier - ReplyTo string // application use - address to to reply to (ex: RPC) - Expiration string // implementation use - message expiration spec - MessageId string // application use - message identifier - Timestamp time.Time // application use - message timestamp - Type string // application use - message type name - UserId string // application use - creating user id - AppId string // application use - creating application + ContentType string `json:"contentType"` // MIME content type + ContentEncoding string `json:"contentEncoding"` // MIME content encoding + Headers Table `json:"headers"` // Application or header exchange table + DeliveryMode uint8 `json:"deliveryMode"` // queue implementation use - Transient (1) or Persistent (2) + Priority uint8 `json:"priority"` // queue implementation use - 0 to 9 + CorrelationId string `json:"correlationId"` // application use - correlation identifier + ReplyTo string `json:"replyTo"` // application use - address to to reply to (ex: RPC) + Expiration string `json:"expiration"` // implementation use - message expiration spec + MessageId string `json:"messageId"` // application use - message identifier + Timestamp time.Time `json:"timestamp"` // application use - message timestamp + Type string `json:"type"` // application use - message type name + UserId string `json:"userId"` // application use - creating user id + AppId string `json:"appId"` // application use - creating application reserved1 string // was cluster-id - process for buffer consumption } diff --git a/tap/extensions/http/handlers.go b/tap/extensions/http/handlers.go index 6b670da8f..ffcbe8c7f 100644 --- a/tap/extensions/http/handlers.go +++ b/tap/extensions/http/handlers.go @@ -7,6 +7,7 @@ import ( "io" "io/ioutil" "net/http" + "strings" "github.com/up9inc/mizu/tap/api" ) @@ -23,8 +24,8 @@ func filterAndEmit(item *api.OutputChannelItem, emitter api.Emitter, options *ap emitter.Emit(item) } -func handleHTTP2Stream(grpcAssembler *GrpcAssembler, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error { - streamID, messageHTTP1, err := grpcAssembler.readMessage() +func handleHTTP2Stream(http2Assembler *Http2Assembler, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error { + streamID, messageHTTP1, isGrpc, err := http2Assembler.readMessage() if err != nil { return err } @@ -34,12 +35,13 @@ func handleHTTP2Stream(grpcAssembler *GrpcAssembler, tcpID *api.TcpID, superTime switch messageHTTP1 := messageHTTP1.(type) { case http.Request: ident := fmt.Sprintf( - "%s->%s %s->%s %d", + "%s->%s %s->%s %d %s", tcpID.SrcIP, tcpID.DstIP, tcpID.SrcPort, tcpID.DstPort, streamID, + "HTTP2", ) item = reqResMatcher.registerRequest(ident, &messageHTTP1, superTimer.CaptureTime) if item != nil { @@ -53,12 +55,13 @@ func handleHTTP2Stream(grpcAssembler *GrpcAssembler, tcpID *api.TcpID, superTime } case http.Response: ident := fmt.Sprintf( - "%s->%s %s->%s %d", + "%s->%s %s->%s %d %s", tcpID.DstIP, tcpID.SrcIP, tcpID.DstPort, tcpID.SrcPort, streamID, + "HTTP2", ) item = reqResMatcher.registerResponse(ident, &messageHTTP1, superTimer.CaptureTime) if item != nil { @@ -73,30 +76,41 @@ func handleHTTP2Stream(grpcAssembler *GrpcAssembler, tcpID *api.TcpID, superTime } if item != nil { - item.Protocol = http2Protocol + if isGrpc { + item.Protocol = grpcProtocol + } else { + item.Protocol = http2Protocol + } filterAndEmit(item, emitter, options) } return nil } -func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error { - req, err := http.ReadRequest(b) +func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) (switchingProtocolsHTTP2 bool, req *http.Request, err error) { + req, err = http.ReadRequest(b) if err != nil { - return err + return } counterPair.Request++ - body, err := ioutil.ReadAll(req.Body) + // Check HTTP2 upgrade - HTTP2 Over Cleartext (H2C) + if strings.Contains(strings.ToLower(req.Header.Get("Connection")), "upgrade") && strings.ToLower(req.Header.Get("Upgrade")) == "h2c" { + switchingProtocolsHTTP2 = true + } + + var body []byte + body, err = ioutil.ReadAll(req.Body) req.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind ident := fmt.Sprintf( - "%s->%s %s->%s %d", + "%s->%s %s->%s %d %s", tcpID.SrcIP, tcpID.DstIP, tcpID.SrcPort, tcpID.DstPort, counterPair.Request, + "HTTP1", ) item := reqResMatcher.registerRequest(ident, req, superTimer.CaptureTime) if item != nil { @@ -109,26 +123,34 @@ func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api } filterAndEmit(item, emitter, options) } - return nil + return } -func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error { - res, err := http.ReadResponse(b, nil) +func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) (switchingProtocolsHTTP2 bool, err error) { + var res *http.Response + res, err = http.ReadResponse(b, nil) if err != nil { - return err + return } counterPair.Response++ - body, err := ioutil.ReadAll(res.Body) + // Check HTTP2 upgrade - HTTP2 Over Cleartext (H2C) + if res.StatusCode == 101 && strings.Contains(strings.ToLower(res.Header.Get("Connection")), "upgrade") && strings.ToLower(res.Header.Get("Upgrade")) == "h2c" { + switchingProtocolsHTTP2 = true + } + + var body []byte + body, err = ioutil.ReadAll(res.Body) res.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind ident := fmt.Sprintf( - "%s->%s %s->%s %d", + "%s->%s %s->%s %d %s", tcpID.DstIP, tcpID.SrcIP, tcpID.DstPort, tcpID.SrcPort, counterPair.Response, + "HTTP1", ) item := reqResMatcher.registerResponse(ident, res, superTimer.CaptureTime) if item != nil { @@ -141,5 +163,5 @@ func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api } filterAndEmit(item, emitter, options) } - return nil + return } diff --git a/tap/extensions/http/helpers.go b/tap/extensions/http/helpers.go new file mode 100644 index 000000000..9348ea3cd --- /dev/null +++ b/tap/extensions/http/helpers.go @@ -0,0 +1,35 @@ +package main + +import ( + "encoding/json" + "fmt" + + "github.com/up9inc/mizu/tap/api" +) + +func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}) { + newMap = make(map[string]interface{}) + for _, header := range mapSlice { + h := header.(map[string]interface{}) + newMap[h["name"].(string)] = h["value"] + } + + return +} + +func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (representation string) { + var table []api.TableData + for _, header := range mapSlice { + h := header.(map[string]interface{}) + selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, h["name"].(string)) + table = append(table, api.TableData{ + Name: h["name"].(string), + Value: h["value"], + Selector: selector, + }) + } + + obj, _ := json.Marshal(table) + representation = string(obj) + return +} diff --git a/tap/extensions/http/grpc_assembler.go b/tap/extensions/http/http2_assembler.go similarity index 80% rename from tap/extensions/http/grpc_assembler.go rename to tap/extensions/http/http2_assembler.go index 0f7a8c00f..0563aacb9 100644 --- a/tap/extensions/http/grpc_assembler.go +++ b/tap/extensions/http/http2_assembler.go @@ -10,6 +10,7 @@ import ( "math" "net/http" "net/url" + "strconv" "strings" "golang.org/x/net/http2" @@ -27,6 +28,26 @@ const protoMinorHTTP2 = 0 var maxHTTP2DataLen = 1 * 1024 * 1024 // 1MB +var grpcStatusCodes = []string{ + "OK", + "CANCELLED", + "UNKNOWN", + "INVALID_ARGUMENT", + "DEADLINE_EXCEEDED", + "NOT_FOUND", + "ALREADY_EXISTS", + "PERMISSION_DENIED", + "RESOURCE_EXHAUSTED", + "FAILED_PRECONDITION", + "ABORTED", + "OUT_OF_RANGE", + "UNIMPLEMENTED", + "INTERNAL", + "UNAVAILABLE", + "DATA_LOSS", + "UNAUTHENTICATED", +} + type messageFragment struct { headers []hpack.HeaderField data []byte @@ -71,37 +92,38 @@ func (fbs *fragmentsByStream) pop(streamID uint32) ([]hpack.HeaderField, []byte) return headers, data } -func createGrpcAssembler(b *bufio.Reader) *GrpcAssembler { +func createHTTP2Assembler(b *bufio.Reader) *Http2Assembler { var framerOutput bytes.Buffer framer := http2.NewFramer(&framerOutput, b) framer.ReadMetaHeaders = hpack.NewDecoder(initialHeaderTableSize, nil) - return &GrpcAssembler{ + return &Http2Assembler{ fragmentsByStream: make(fragmentsByStream), framer: framer, } } -type GrpcAssembler struct { +type Http2Assembler struct { fragmentsByStream fragmentsByStream framer *http2.Framer } -func (ga *GrpcAssembler) readMessage() (uint32, interface{}, error) { +func (ga *Http2Assembler) readMessage() (streamID uint32, messageHTTP1 interface{}, isGrpc bool, err error) { // Exactly one Framer is used for each half connection. // (Instead of creating a new Framer for each ReadFrame operation) // This is needed in order to decompress the headers, // because the compression context is updated with each requests/response. frame, err := ga.framer.ReadFrame() if err != nil { - return 0, nil, err + return } - streamID := frame.Header().StreamID + streamID = frame.Header().StreamID ga.fragmentsByStream.appendFrame(streamID, frame) if !(ga.isStreamEnd(frame)) { - return 0, nil, nil + streamID = 0 + return } headers, data := ga.fragmentsByStream.pop(streamID) @@ -115,13 +137,29 @@ func (ga *GrpcAssembler) readMessage() (uint32, interface{}, error) { dataString := base64.StdEncoding.EncodeToString(data) // Use http1 types only because they are expected in http_matcher. - // TODO: Create an interface that will be used by http_matcher:registerRequest and http_matcher:registerRequest - // to accept both HTTP/1.x and HTTP/2 requests and responses - var messageHTTP1 interface{} - if _, ok := headersHTTP1[":method"]; ok { + method := headersHTTP1.Get(":method") + status := headersHTTP1.Get(":status") + + // gRPC detection + grpcStatus := headersHTTP1.Get("Grpc-Status") + if grpcStatus != "" { + isGrpc = true + status = grpcStatus + } + + if strings.Contains(headersHTTP1.Get("Content-Type"), "application/grpc") { + isGrpc = true + grpcPath := headersHTTP1.Get(":path") + pathSegments := strings.Split(grpcPath, "/") + if len(pathSegments) > 0 { + method = pathSegments[len(pathSegments)-1] + } + } + + if method != "" { messageHTTP1 = http.Request{ URL: &url.URL{}, - Method: "POST", + Method: method, Header: headersHTTP1, Proto: protoHTTP2, ProtoMajor: protoMajorHTTP2, @@ -129,8 +167,16 @@ func (ga *GrpcAssembler) readMessage() (uint32, interface{}, error) { Body: io.NopCloser(strings.NewReader(dataString)), ContentLength: int64(len(dataString)), } - } else if _, ok := headersHTTP1[":status"]; ok { + } else if status != "" { + var statusCode int + + statusCode, err = strconv.Atoi(status) + if err != nil { + return + } + messageHTTP1 = http.Response{ + StatusCode: statusCode, Header: headersHTTP1, Proto: protoHTTP2, ProtoMajor: protoMajorHTTP2, @@ -139,13 +185,14 @@ func (ga *GrpcAssembler) readMessage() (uint32, interface{}, error) { ContentLength: int64(len(dataString)), } } else { - return 0, nil, errors.New("failed to assemble stream: neither a request nor a message") + err = errors.New("failed to assemble stream: neither a request nor a message") + return } - return streamID, messageHTTP1, nil + return } -func (ga *GrpcAssembler) isStreamEnd(frame http2.Frame) bool { +func (ga *Http2Assembler) isStreamEnd(frame http2.Frame) bool { switch frame := frame.(type) { case *http2.MetaHeadersFrame: if frame.StreamEnded() { diff --git a/tap/extensions/http/main.go b/tap/extensions/http/main.go index 3ca432a71..26cf40c8c 100644 --- a/tap/extensions/http/main.go +++ b/tap/extensions/http/main.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "log" + "net/http" "net/url" "time" @@ -17,25 +18,41 @@ var protocol api.Protocol = api.Protocol{ Name: "http", LongName: "Hypertext Transfer Protocol -- HTTP/1.1", Abbreviation: "HTTP", + Macro: "http", Version: "1.1", BackgroundColor: "#205cf5", ForegroundColor: "#ffffff", FontSize: 12, ReferenceLink: "https://datatracker.ietf.org/doc/html/rfc2616", - Ports: []string{"80", "8080", "50051"}, + Ports: []string{"80", "443", "8080"}, Priority: 0, } var http2Protocol api.Protocol = api.Protocol{ Name: "http", - LongName: "Hypertext Transfer Protocol Version 2 (HTTP/2) (gRPC)", + LongName: "Hypertext Transfer Protocol Version 2 (HTTP/2)", Abbreviation: "HTTP/2", + Macro: "http2", Version: "2.0", BackgroundColor: "#244c5a", ForegroundColor: "#ffffff", FontSize: 11, ReferenceLink: "https://datatracker.ietf.org/doc/html/rfc7540", - Ports: []string{"80", "8080"}, + Ports: []string{"80", "443", "8080"}, + Priority: 0, +} + +var grpcProtocol api.Protocol = api.Protocol{ + Name: "http", + LongName: "Hypertext Transfer Protocol Version 2 (HTTP/2) [ gRPC over HTTP/2 ]", + Abbreviation: "gRPC", + Macro: "grpc", + Version: "2.0", + BackgroundColor: "#244c5a", + ForegroundColor: "#ffffff", + FontSize: 11, + ReferenceLink: "https://grpc.github.io/grpc/core/md_doc_statuscodes.html", + Ports: []string{"80", "443", "8080", "50051"}, Priority: 0, } @@ -56,26 +73,34 @@ func (d dissecting) Register(extension *api.Extension) { } func (d dissecting) Ping() { - log.Printf("pong %s\n", protocol.Name) + log.Printf("pong %s", protocol.Name) } func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error { isHTTP2, err := checkIsHTTP2Connection(b, isClient) - var grpcAssembler *GrpcAssembler + var http2Assembler *Http2Assembler if isHTTP2 { prepareHTTP2Connection(b, isClient) - grpcAssembler = createGrpcAssembler(b) + http2Assembler = createHTTP2Assembler(b) } dissected := false + switchingProtocolsHTTP2 := false for { + if switchingProtocolsHTTP2 { + switchingProtocolsHTTP2 = false + isHTTP2, err = checkIsHTTP2Connection(b, isClient) + prepareHTTP2Connection(b, isClient) + http2Assembler = createHTTP2Assembler(b) + } + if superIdentifier.Protocol != nil && superIdentifier.Protocol != &protocol { return errors.New("Identified by another protocol") } if isHTTP2 { - err = handleHTTP2Stream(grpcAssembler, tcpID, superTimer, emitter, options) + err = handleHTTP2Stream(http2Assembler, tcpID, superTimer, emitter, options) if err == io.EOF || err == io.ErrUnexpectedEOF { break } else if err != nil { @@ -83,15 +108,39 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co } dissected = true } else if isClient { - err = handleHTTP1ClientStream(b, tcpID, counterPair, superTimer, emitter, options) + var req *http.Request + switchingProtocolsHTTP2, req, err = handleHTTP1ClientStream(b, tcpID, counterPair, superTimer, emitter, options) if err == io.EOF || err == io.ErrUnexpectedEOF { break } else if err != nil { continue } dissected = true + + // In case of an HTTP2 upgrade, duplicate the HTTP1 request into HTTP2 with stream ID 1 + if switchingProtocolsHTTP2 { + ident := fmt.Sprintf( + "%s->%s %s->%s 1 %s", + tcpID.SrcIP, + tcpID.DstIP, + tcpID.SrcPort, + tcpID.DstPort, + "HTTP2", + ) + item := reqResMatcher.registerRequest(ident, req, superTimer.CaptureTime) + if item != nil { + item.ConnectionInfo = &api.ConnectionInfo{ + ClientIP: tcpID.SrcIP, + ClientPort: tcpID.SrcPort, + ServerIP: tcpID.DstIP, + ServerPort: tcpID.DstPort, + IsOutgoing: true, + } + filterAndEmit(item, emitter, options) + } + } } else { - err = handleHTTP1ServerStream(b, tcpID, counterPair, superTimer, emitter, options) + switchingProtocolsHTTP2, err = handleHTTP1ServerStream(b, tcpID, counterPair, superTimer, emitter, options) if err == io.EOF || err == io.ErrUnexpectedEOF { break } else if err != nil { @@ -108,23 +157,16 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co return nil } -func SetHostname(address, newHostname string) string { - replacedUrl, err := url.Parse(address) - if err != nil { - return address - } - replacedUrl.Host = newHostname - return replacedUrl.String() -} - -func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry { - var host, scheme, authority, path, service string +func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry { + var host, authority, path string request := item.Pair.Request.Payload.(map[string]interface{}) response := item.Pair.Response.Payload.(map[string]interface{}) reqDetails := request["details"].(map[string]interface{}) resDetails := response["details"].(map[string]interface{}) + isRequestUpgradedH2C := false + for _, header := range reqDetails["headers"].([]interface{}) { h := header.(map[string]interface{}) if h["name"] == "Host" { @@ -133,83 +175,111 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve if h["name"] == ":authority" { authority = h["value"].(string) } - if h["name"] == ":scheme" { - scheme = h["value"].(string) - } if h["name"] == ":path" { path = h["value"].(string) } + + if h["name"] == "Upgrade" { + if h["value"].(string) == "h2c" { + isRequestUpgradedH2C = true + } + } } - if item.Protocol.Version == "2.0" { - service = fmt.Sprintf("%s://%s", scheme, authority) + if resDetails["bodySize"].(float64) < 0 { + resDetails["bodySize"] = 0 + } + + if item.Protocol.Version == "2.0" && !isRequestUpgradedH2C { + if resolvedDestination == "" { + resolvedDestination = authority + } + if resolvedDestination == "" { + resolvedDestination = host + } } else { - service = fmt.Sprintf("http://%s", host) - path = reqDetails["url"].(string) + u, err := url.Parse(reqDetails["url"].(string)) + if err != nil { + path = reqDetails["url"].(string) + } else { + path = u.Path + } } - request["url"] = path - if resolvedDestination != "" { - service = SetHostname(service, resolvedDestination) - } else if resolvedSource != "" { - service = SetHostname(service, resolvedSource) + request["url"] = reqDetails["url"].(string) + reqDetails["targetUri"] = reqDetails["url"] + reqDetails["path"] = path + reqDetails["summary"] = path + + // Rearrange the maps for the querying + reqDetails["_headers"] = reqDetails["headers"] + reqDetails["headers"] = mapSliceRebuildAsMap(reqDetails["_headers"].([]interface{})) + resDetails["_headers"] = resDetails["headers"] + resDetails["headers"] = mapSliceRebuildAsMap(resDetails["_headers"].([]interface{})) + + reqDetails["_cookies"] = reqDetails["cookies"] + reqDetails["cookies"] = mapSliceRebuildAsMap(reqDetails["_cookies"].([]interface{})) + resDetails["_cookies"] = resDetails["cookies"] + resDetails["cookies"] = mapSliceRebuildAsMap(resDetails["_cookies"].([]interface{})) + + reqDetails["_queryString"] = reqDetails["queryString"] + reqDetails["queryString"] = mapSliceRebuildAsMap(reqDetails["_queryString"].([]interface{})) + + method := reqDetails["method"].(string) + statusCode := int(resDetails["status"].(float64)) + if item.Protocol.Abbreviation == "gRPC" { + resDetails["statusText"] = grpcStatusCodes[statusCode] + } + + if item.Protocol.Version == "2.0" && !isRequestUpgradedH2C { + reqDetails["url"] = path + request["url"] = path } elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds() - entryBytes, _ := json.Marshal(item.Pair) + if elapsedTime < 0 { + elapsedTime = 0 + } + httpPair, _ := json.Marshal(item.Pair) return &api.MizuEntry{ - ProtocolName: protocol.Name, - ProtocolLongName: protocol.LongName, - ProtocolAbbreviation: protocol.Abbreviation, - ProtocolVersion: item.Protocol.Version, - ProtocolBackgroundColor: protocol.BackgroundColor, - ProtocolForegroundColor: protocol.ForegroundColor, - ProtocolFontSize: protocol.FontSize, - ProtocolReferenceLink: protocol.ReferenceLink, - EntryId: entryId, - Entry: string(entryBytes), - Url: fmt.Sprintf("%s%s", service, path), - Method: reqDetails["method"].(string), - Status: int(resDetails["status"].(float64)), - RequestSenderIp: item.ConnectionInfo.ClientIP, - Service: service, - Timestamp: item.Timestamp, - ElapsedTime: elapsedTime, - Path: path, - ResolvedSource: resolvedSource, - ResolvedDestination: resolvedDestination, - SourceIp: item.ConnectionInfo.ClientIP, - DestinationIp: item.ConnectionInfo.ServerIP, - SourcePort: item.ConnectionInfo.ClientPort, - DestinationPort: item.ConnectionInfo.ServerPort, - IsOutgoing: item.ConnectionInfo.IsOutgoing, + Protocol: item.Protocol, + Source: &api.TCP{ + Name: resolvedSource, + IP: item.ConnectionInfo.ClientIP, + Port: item.ConnectionInfo.ClientPort, + }, + Destination: &api.TCP{ + Name: resolvedDestination, + IP: item.ConnectionInfo.ServerIP, + Port: item.ConnectionInfo.ServerPort, + }, + Outgoing: item.ConnectionInfo.IsOutgoing, + Request: reqDetails, + Response: resDetails, + Method: method, + Status: statusCode, + Timestamp: item.Timestamp, + StartTime: item.Pair.Request.CaptureTime, + ElapsedTime: elapsedTime, + Summary: path, + IsOutgoing: item.ConnectionInfo.IsOutgoing, + HTTPPair: string(httpPair), } } func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { - var p api.Protocol - if entry.ProtocolVersion == "2.0" { - p = http2Protocol - } else { - p = protocol - } return &api.BaseEntryDetails{ - Id: entry.EntryId, - Protocol: p, - Url: entry.Url, - RequestSenderIp: entry.RequestSenderIp, - Service: entry.Service, - Path: entry.Path, - Summary: entry.Path, - StatusCode: entry.Status, - Method: entry.Method, - Timestamp: entry.Timestamp, - SourceIp: entry.SourceIp, - DestinationIp: entry.DestinationIp, - SourcePort: entry.SourcePort, - DestinationPort: entry.DestinationPort, - IsOutgoing: entry.IsOutgoing, - Latency: entry.ElapsedTime, + Id: entry.Id, + Protocol: entry.Protocol, + Path: entry.Path, + Summary: entry.Summary, + StatusCode: entry.Status, + Method: entry.Method, + Timestamp: entry.Timestamp, + Source: entry.Source, + Destination: entry.Destination, + IsOutgoing: entry.IsOutgoing, + Latency: entry.ElapsedTime, Rules: api.ApplicableRules{ Latency: 0, Status: false, @@ -218,45 +288,50 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { } func representRequest(request map[string]interface{}) (repRequest []interface{}) { - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Method", - "value": request["method"].(string), + Name: "Method", + Value: request["method"].(string), + Selector: `request.method`, }, { - "name": "URL", - "value": request["url"].(string), + Name: "Target URI", + Value: request["targetUri"].(string), + Selector: `request.targetUri`, }, { - "name": "Body Size", - "value": fmt.Sprintf("%g bytes", request["bodySize"].(float64)), + Name: "Path", + Value: request["path"].(string), + Selector: `request.path`, + }, + { + Name: "Body Size (bytes)", + Value: int64(request["bodySize"].(float64)), + Selector: `request.bodySize`, }, }) - repRequest = append(repRequest, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + repRequest = append(repRequest, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - headers, _ := json.Marshal(request["headers"].([]interface{})) - repRequest = append(repRequest, map[string]string{ - "type": api.TABLE, - "title": "Headers", - "data": string(headers), + repRequest = append(repRequest, api.SectionData{ + Type: api.TABLE, + Title: "Headers", + Data: representMapSliceAsTable(request["_headers"].([]interface{}), `request.headers`), }) - cookies, _ := json.Marshal(request["cookies"].([]interface{})) - repRequest = append(repRequest, map[string]string{ - "type": api.TABLE, - "title": "Cookies", - "data": string(cookies), + repRequest = append(repRequest, api.SectionData{ + Type: api.TABLE, + Title: "Cookies", + Data: representMapSliceAsTable(request["_cookies"].([]interface{}), `request.cookies`), }) - queryString, _ := json.Marshal(request["queryString"].([]interface{})) - repRequest = append(repRequest, map[string]string{ - "type": api.TABLE, - "title": "Query String", - "data": string(queryString), + repRequest = append(repRequest, api.SectionData{ + Type: api.TABLE, + Title: "Query String", + Data: representMapSliceAsTable(request["_queryString"].([]interface{}), `request.queryString`), }) postData, _ := request["postData"].(map[string]interface{}) @@ -266,12 +341,12 @@ func representRequest(request map[string]interface{}) (repRequest []interface{}) } text, _ := postData["text"] if text != nil { - repRequest = append(repRequest, map[string]string{ - "type": api.BODY, - "title": "POST Data (text/plain)", - "encoding": "", - "mime_type": mimeType.(string), - "data": text.(string), + repRequest = append(repRequest, api.SectionData{ + Type: api.BODY, + Title: "POST Data (text/plain)", + MimeType: mimeType.(string), + Data: text.(string), + Selector: `request.postData.text`, }) } @@ -285,16 +360,16 @@ func representRequest(request map[string]interface{}) (repRequest []interface{}) "value": string(params), }, }) - repRequest = append(repRequest, map[string]string{ - "type": api.TABLE, - "title": "POST Data (multipart/form-data)", - "data": string(multipart), + repRequest = append(repRequest, api.SectionData{ + Type: api.TABLE, + Title: "POST Data (multipart/form-data)", + Data: string(multipart), }) } else { - repRequest = append(repRequest, map[string]string{ - "type": api.TABLE, - "title": "POST Data (application/x-www-form-urlencoded)", - "data": string(params), + repRequest = append(repRequest, api.SectionData{ + Type: api.TABLE, + Title: "POST Data (application/x-www-form-urlencoded)", + Data: representMapSliceAsTable(postData["params"].([]interface{}), `request.postData.params`), }) } } @@ -308,38 +383,39 @@ func representResponse(response map[string]interface{}) (repResponse []interface bodySize = int64(response["bodySize"].(float64)) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Status", - "value": fmt.Sprintf("%g", response["status"].(float64)), + Name: "Status", + Value: int64(response["status"].(float64)), + Selector: `response.status`, }, { - "name": "Status Text", - "value": response["statusText"].(string), + Name: "Status Text", + Value: response["statusText"].(string), + Selector: `response.statusText`, }, { - "name": "Body Size", - "value": fmt.Sprintf("%d bytes", bodySize), + Name: "Body Size (bytes)", + Value: bodySize, + Selector: `response.bodySize`, }, }) - repResponse = append(repResponse, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + repResponse = append(repResponse, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - headers, _ := json.Marshal(response["headers"].([]interface{})) - repResponse = append(repResponse, map[string]string{ - "type": api.TABLE, - "title": "Headers", - "data": string(headers), + repResponse = append(repResponse, api.SectionData{ + Type: api.TABLE, + Title: "Headers", + Data: representMapSliceAsTable(response["_headers"].([]interface{}), `response.headers`), }) - cookies, _ := json.Marshal(response["cookies"].([]interface{})) - repResponse = append(repResponse, map[string]string{ - "type": api.TABLE, - "title": "Cookies", - "data": string(cookies), + repResponse = append(repResponse, api.SectionData{ + Type: api.TABLE, + Title: "Cookies", + Data: representMapSliceAsTable(response["_cookies"].([]interface{}), `response.cookies`), }) content, _ := response["content"].(map[string]interface{}) @@ -350,37 +426,35 @@ func representResponse(response map[string]interface{}) (repResponse []interface encoding, _ := content["encoding"] text, _ := content["text"] if text != nil { - repResponse = append(repResponse, map[string]string{ - "type": api.BODY, - "title": "Body", - "encoding": encoding.(string), - "mime_type": mimeType.(string), - "data": text.(string), + repResponse = append(repResponse, api.SectionData{ + Type: api.BODY, + Title: "Body", + Encoding: encoding.(string), + MimeType: mimeType.(string), + Data: text.(string), + Selector: `response.content.text`, }) } return } -func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) { - if entry.ProtocolVersion == "2.0" { - p = http2Protocol - } else { - p = protocol - } - var root map[string]interface{} - json.Unmarshal([]byte(entry.Entry), &root) +func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) { representation := make(map[string]interface{}, 0) - request := root["request"].(map[string]interface{})["payload"].(map[string]interface{}) - response := root["response"].(map[string]interface{})["payload"].(map[string]interface{}) - reqDetails := request["details"].(map[string]interface{}) - resDetails := response["details"].(map[string]interface{}) - repRequest := representRequest(reqDetails) - repResponse, bodySize := representResponse(resDetails) + repRequest := representRequest(request) + repResponse, bodySize := representResponse(response) representation["request"] = repRequest representation["response"] = repResponse object, err = json.Marshal(representation) return } +func (d dissecting) Macros() map[string]string { + return map[string]string{ + `http`: fmt.Sprintf(`proto.name == "%s" and proto.version == "%s"`, protocol.Name, protocol.Version), + `http2`: fmt.Sprintf(`proto.name == "%s" and proto.version == "%s"`, protocol.Name, http2Protocol.Version), + `grpc`: fmt.Sprintf(`proto.name == "%s" and proto.version == "%s" and proto.macro == "%s"`, protocol.Name, grpcProtocol.Version, grpcProtocol.Macro), + } +} + var Dissector dissecting diff --git a/tap/extensions/http/matcher.go b/tap/extensions/http/matcher.go index 08a0b4e5c..01048fa21 100644 --- a/tap/extensions/http/matcher.go +++ b/tap/extensions/http/matcher.go @@ -92,6 +92,6 @@ func splitIdent(ident string) []string { } func genKey(split []string) string { - key := fmt.Sprintf("%s:%s->%s:%s,%s", split[0], split[2], split[1], split[3], split[4]) + key := fmt.Sprintf("%s:%s->%s:%s,%s%s", split[0], split[2], split[1], split[3], split[4], split[5]) return key } diff --git a/tap/extensions/http/sensitive_data_cleaner.go b/tap/extensions/http/sensitive_data_cleaner.go index 4d4487333..8dcf3c2cc 100644 --- a/tap/extensions/http/sensitive_data_cleaner.go +++ b/tap/extensions/http/sensitive_data_cleaner.go @@ -16,6 +16,7 @@ import ( ) const maskedFieldPlaceholderValue = "[REDACTED]" +const userAgent = "user-agent" //these values MUST be all lower case and contain no `-` or `_` characters var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password", @@ -32,7 +33,7 @@ func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteri request := item.Pair.Request.Payload.(api.HTTPPayload).Data.(*http.Request) for headerKey, headerValues := range request.Header { - if strings.ToLower(headerKey) == "user-agent" { + if strings.ToLower(headerKey) == userAgent { for _, userAgent := range options.IgnoredUserAgents { for _, headerValue := range headerValues { if strings.Contains(strings.ToLower(headerValue), strings.ToLower(userAgent)) { @@ -89,6 +90,10 @@ func filterResponseBody(response *http.Response, options *api.TrafficFilteringOp func filterHeaders(headers *http.Header) { for key, _ := range *headers { + if strings.ToLower(key) == userAgent { + continue + } + if strings.ToLower(key) == "cookie" { headers.Del(key) } else if isFieldNameSensitive(key) { diff --git a/tap/extensions/kafka/helpers.go b/tap/extensions/kafka/helpers.go index 3a9a238eb..dffe30bd8 100644 --- a/tap/extensions/kafka/helpers.go +++ b/tap/extensions/kafka/helpers.go @@ -27,48 +27,54 @@ type KafkaWrapper struct { } func representRequestHeader(data map[string]interface{}, rep []interface{}) []interface{} { - requestHeader, _ := json.Marshal([]map[string]string{ + requestHeader, _ := json.Marshal([]api.TableData{ { - "name": "ApiKey", - "value": apiNames[int(data["ApiKey"].(float64))], + Name: "ApiKey", + Value: apiNames[int(data["apiKey"].(float64))], + Selector: `request.apiKey`, }, { - "name": "ApiVersion", - "value": fmt.Sprintf("%d", int(data["ApiVersion"].(float64))), + Name: "ApiVersion", + Value: fmt.Sprintf("%d", int(data["apiVersion"].(float64))), + Selector: `request.apiVersion`, }, { - "name": "Client ID", - "value": data["ClientID"].(string), + Name: "Client ID", + Value: data["clientID"].(string), + Selector: `request.clientID`, }, { - "name": "Correlation ID", - "value": fmt.Sprintf("%d", int(data["CorrelationID"].(float64))), + Name: "Correlation ID", + Value: fmt.Sprintf("%d", int(data["correlationID"].(float64))), + Selector: `request.correlationID`, }, { - "name": "Size", - "value": fmt.Sprintf("%d", int(data["Size"].(float64))), + Name: "Size", + Value: fmt.Sprintf("%d", int(data["size"].(float64))), + Selector: `request.size`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Request Header", - "data": string(requestHeader), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Request Header", + Data: string(requestHeader), }) return rep } func representResponseHeader(data map[string]interface{}, rep []interface{}) []interface{} { - requestHeader, _ := json.Marshal([]map[string]string{ + requestHeader, _ := json.Marshal([]api.TableData{ { - "name": "Correlation ID", - "value": fmt.Sprintf("%d", int(data["CorrelationID"].(float64))), + Name: "Correlation ID", + Value: fmt.Sprintf("%d", int(data["correlationID"].(float64))), + Selector: `response.correlationID`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Response Header", - "data": string(requestHeader), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Response Header", + Data: string(requestHeader), }) return rep @@ -79,46 +85,50 @@ func representMetadataRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) topics := "" allowAutoTopicCreation := "" includeClusterAuthorizedOperations := "" includeTopicAuthorizedOperations := "" - if payload["Topics"] != nil { - x, _ := json.Marshal(payload["Topics"].([]interface{})) + if payload["topics"] != nil { + x, _ := json.Marshal(payload["topics"].([]interface{})) topics = string(x) } - if payload["AllowAutoTopicCreation"] != nil { - allowAutoTopicCreation = strconv.FormatBool(payload["AllowAutoTopicCreation"].(bool)) + if payload["allowAutoTopicCreation"] != nil { + allowAutoTopicCreation = strconv.FormatBool(payload["allowAutoTopicCreation"].(bool)) } - if payload["IncludeClusterAuthorizedOperations"] != nil { - includeClusterAuthorizedOperations = strconv.FormatBool(payload["IncludeClusterAuthorizedOperations"].(bool)) + if payload["includeClusterAuthorizedOperations"] != nil { + includeClusterAuthorizedOperations = strconv.FormatBool(payload["includeClusterAuthorizedOperations"].(bool)) } - if payload["IncludeTopicAuthorizedOperations"] != nil { - includeTopicAuthorizedOperations = strconv.FormatBool(payload["IncludeTopicAuthorizedOperations"].(bool)) + if payload["includeTopicAuthorizedOperations"] != nil { + includeTopicAuthorizedOperations = strconv.FormatBool(payload["includeTopicAuthorizedOperations"].(bool)) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Topics", - "value": topics, + Name: "Topics", + Value: topics, + Selector: `request.payload.topics`, }, { - "name": "Allow Auto Topic Creation", - "value": allowAutoTopicCreation, + Name: "Allow Auto Topic Creation", + Value: allowAutoTopicCreation, + Selector: `request.payload.allowAutoTopicCreation`, }, { - "name": "Include Cluster Authorized Operations", - "value": includeClusterAuthorizedOperations, + Name: "Include Cluster Authorized Operations", + Value: includeClusterAuthorizedOperations, + Selector: `request.payload.includeClusterAuthorizedOperations`, }, { - "name": "Include Topic Authorized Operations", - "value": includeTopicAuthorizedOperations, + Name: "Include Topic Authorized Operations", + Value: includeTopicAuthorizedOperations, + Selector: `request.payload.includeTopicAuthorizedOperations`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -129,63 +139,69 @@ func representMetadataResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) topics := "" - if payload["Topics"] != nil { - _topics, _ := json.Marshal(payload["Topics"].([]interface{})) + if payload["topics"] != nil { + _topics, _ := json.Marshal(payload["topics"].([]interface{})) topics = string(_topics) } brokers := "" - if payload["Brokers"] != nil { - _brokers, _ := json.Marshal(payload["Brokers"].([]interface{})) + if payload["brokers"] != nil { + _brokers, _ := json.Marshal(payload["brokers"].([]interface{})) brokers = string(_brokers) } controllerID := "" clusterID := "" throttleTimeMs := "" clusterAuthorizedOperations := "" - if payload["ControllerID"] != nil { - controllerID = fmt.Sprintf("%d", int(payload["ControllerID"].(float64))) + if payload["controllerID"] != nil { + controllerID = fmt.Sprintf("%d", int(payload["controllerID"].(float64))) } - if payload["ClusterID"] != nil { - clusterID = payload["ClusterID"].(string) + if payload["clusterID"] != nil { + clusterID = payload["clusterID"].(string) } - if payload["ThrottleTimeMs"] != nil { - throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64))) + if payload["throttleTimeMs"] != nil { + throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) } - if payload["ClusterAuthorizedOperations"] != nil { - clusterAuthorizedOperations = fmt.Sprintf("%d", int(payload["ClusterAuthorizedOperations"].(float64))) + if payload["clusterAuthorizedOperations"] != nil { + clusterAuthorizedOperations = fmt.Sprintf("%d", int(payload["clusterAuthorizedOperations"].(float64))) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Throttle Time (ms)", - "value": throttleTimeMs, + Name: "Throttle Time (ms)", + Value: throttleTimeMs, + Selector: `response.payload.throttleTimeMs`, }, { - "name": "Brokers", - "value": brokers, + Name: "Brokers", + Value: brokers, + Selector: `response.payload.brokers`, }, { - "name": "Cluster ID", - "value": clusterID, + Name: "Cluster ID", + Value: clusterID, + Selector: `response.payload.clusterID`, }, { - "name": "Controller ID", - "value": controllerID, + Name: "Controller ID", + Value: controllerID, + Selector: `response.payload.controllerID`, }, { - "name": "Topics", - "value": topics, + Name: "Topics", + Value: topics, + Selector: `response.payload.topics`, }, { - "name": "Cluster Authorized Operations", - "value": clusterAuthorizedOperations, + Name: "Cluster Authorized Operations", + Value: clusterAuthorizedOperations, + Selector: `response.payload.clusterAuthorizedOperations`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -196,29 +212,31 @@ func representApiVersionsRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) clientSoftwareName := "" clientSoftwareVersion := "" - if payload["ClientSoftwareName"] != nil { - clientSoftwareName = payload["ClientSoftwareName"].(string) + if payload["clientSoftwareName"] != nil { + clientSoftwareName = payload["clientSoftwareName"].(string) } - if payload["ClientSoftwareVersion"] != nil { - clientSoftwareVersion = payload["ClientSoftwareVersion"].(string) + if payload["clientSoftwareVersion"] != nil { + clientSoftwareVersion = payload["clientSoftwareVersion"].(string) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Client Software Name", - "value": clientSoftwareName, + Name: "Client Software Name", + Value: clientSoftwareName, + Selector: `request.payload.clientSoftwareName`, }, { - "name": "Client Software Version", - "value": clientSoftwareVersion, + Name: "Client Software Version", + Value: clientSoftwareVersion, + Selector: `request.payload.clientSoftwareVersion`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -229,34 +247,37 @@ func representApiVersionsResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) apiKeys := "" - if payload["TopicNames"] != nil { - x, _ := json.Marshal(payload["ApiKeys"].([]interface{})) + if payload["apiKeys"] != nil { + x, _ := json.Marshal(payload["apiKeys"].([]interface{})) apiKeys = string(x) } throttleTimeMs := "" - if payload["ThrottleTimeMs"] != nil { - throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64))) + if payload["throttleTimeMs"] != nil { + throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Error Code", - "value": fmt.Sprintf("%d", int(payload["ErrorCode"].(float64))), + Name: "Error Code", + Value: fmt.Sprintf("%d", int(payload["errorCode"].(float64))), + Selector: `response.payload.errorCode`, }, { - "name": "ApiKeys", - "value": apiKeys, + Name: "ApiKeys", + Value: apiKeys, + Selector: `response.payload.apiKeys`, }, { - "name": "Throttle Time (ms)", - "value": throttleTimeMs, + Name: "Throttle Time (ms)", + Value: throttleTimeMs, + Selector: `response.payload.throttleTimeMs`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -267,39 +288,43 @@ func representProduceRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) topicData := "" - _topicData := payload["TopicData"] + _topicData := payload["topicData"] if _topicData != nil { x, _ := json.Marshal(_topicData.([]interface{})) topicData = string(x) } transactionalID := "" - if payload["TransactionalID"] != nil { - transactionalID = payload["TransactionalID"].(string) + if payload["transactionalID"] != nil { + transactionalID = payload["transactionalID"].(string) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Transactional ID", - "value": transactionalID, + Name: "Transactional ID", + Value: transactionalID, + Selector: `request.payload.transactionalID`, }, { - "name": "Required Acknowledgements", - "value": fmt.Sprintf("%d", int(payload["RequiredAcks"].(float64))), + Name: "Required Acknowledgements", + Value: fmt.Sprintf("%d", int(payload["requiredAcks"].(float64))), + Selector: `request.payload.requiredAcks`, }, { - "name": "Timeout", - "value": fmt.Sprintf("%d", int(payload["Timeout"].(float64))), + Name: "Timeout", + Value: fmt.Sprintf("%d", int(payload["timeout"].(float64))), + Selector: `request.payload.timeout`, }, { - "name": "Topic Data", - "value": topicData, + Name: "Topic Data", + Value: topicData, + Selector: `request.payload.topicData`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -310,30 +335,32 @@ func representProduceResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) responses := "" - if payload["Responses"] != nil { - _responses, _ := json.Marshal(payload["Responses"].([]interface{})) + if payload["responses"] != nil { + _responses, _ := json.Marshal(payload["responses"].([]interface{})) responses = string(_responses) } throttleTimeMs := "" - if payload["ThrottleTimeMs"] != nil { - throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64))) + if payload["throttleTimeMs"] != nil { + throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Responses", - "value": string(responses), + Name: "Responses", + Value: string(responses), + Selector: `response.payload.responses`, }, { - "name": "Throttle Time (ms)", - "value": throttleTimeMs, + Name: "Throttle Time (ms)", + Value: throttleTimeMs, + Selector: `response.payload.throttleTimeMs`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -344,87 +371,97 @@ func representFetchRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) topics := "" - if payload["Topics"] != nil { - _topics, _ := json.Marshal(payload["Topics"].([]interface{})) + if payload["topics"] != nil { + _topics, _ := json.Marshal(payload["topics"].([]interface{})) topics = string(_topics) } replicaId := "" - if payload["ReplicaId"] != nil { - replicaId = fmt.Sprintf("%d", int(payload["ReplicaId"].(float64))) + if payload["replicaId"] != nil { + replicaId = fmt.Sprintf("%d", int(payload["replicaId"].(float64))) } maxBytes := "" - if payload["MaxBytes"] != nil { - maxBytes = fmt.Sprintf("%d", int(payload["MaxBytes"].(float64))) + if payload["maxBytes"] != nil { + maxBytes = fmt.Sprintf("%d", int(payload["maxBytes"].(float64))) } isolationLevel := "" - if payload["IsolationLevel"] != nil { - isolationLevel = fmt.Sprintf("%d", int(payload["IsolationLevel"].(float64))) + if payload["isolationLevel"] != nil { + isolationLevel = fmt.Sprintf("%d", int(payload["isolationLevel"].(float64))) } sessionId := "" - if payload["SessionId"] != nil { - sessionId = fmt.Sprintf("%d", int(payload["SessionId"].(float64))) + if payload["sessionId"] != nil { + sessionId = fmt.Sprintf("%d", int(payload["sessionId"].(float64))) } sessionEpoch := "" - if payload["SessionEpoch"] != nil { - sessionEpoch = fmt.Sprintf("%d", int(payload["SessionEpoch"].(float64))) + if payload["sessionEpoch"] != nil { + sessionEpoch = fmt.Sprintf("%d", int(payload["sessionEpoch"].(float64))) } forgottenTopicsData := "" - if payload["ForgottenTopicsData"] != nil { - x, _ := json.Marshal(payload["ForgottenTopicsData"].(map[string]interface{})) + if payload["forgottenTopicsData"] != nil { + x, _ := json.Marshal(payload["forgottenTopicsData"].(map[string]interface{})) forgottenTopicsData = string(x) } rackId := "" - if payload["RackId"] != nil { - rackId = payload["RackId"].(string) + if payload["rackId"] != nil { + rackId = payload["rackId"].(string) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Replica ID", - "value": replicaId, + Name: "Replica ID", + Value: replicaId, + Selector: `request.payload.replicaId`, }, { - "name": "Maximum Wait (ms)", - "value": fmt.Sprintf("%d", int(payload["MaxWaitMs"].(float64))), + Name: "Maximum Wait (ms)", + Value: fmt.Sprintf("%d", int(payload["maxWaitMs"].(float64))), + Selector: `request.payload.maxWaitMs`, }, { - "name": "Minimum Bytes", - "value": fmt.Sprintf("%d", int(payload["MinBytes"].(float64))), + Name: "Minimum Bytes", + Value: fmt.Sprintf("%d", int(payload["minBytes"].(float64))), + Selector: `request.payload.minBytes`, }, { - "name": "Maximum Bytes", - "value": maxBytes, + Name: "Maximum Bytes", + Value: maxBytes, + Selector: `request.payload.maxBytes`, }, { - "name": "Isolation Level", - "value": isolationLevel, + Name: "Isolation Level", + Value: isolationLevel, + Selector: `request.payload.isolationLevel`, }, { - "name": "Session ID", - "value": sessionId, + Name: "Session ID", + Value: sessionId, + Selector: `request.payload.sessionId`, }, { - "name": "Session Epoch", - "value": sessionEpoch, + Name: "Session Epoch", + Value: sessionEpoch, + Selector: `request.payload.sessionEpoch`, }, { - "name": "Topics", - "value": topics, + Name: "Topics", + Value: topics, + Selector: `request.payload.topics`, }, { - "name": "Forgotten Topics Data", - "value": forgottenTopicsData, + Name: "Forgotten Topics Data", + Value: forgottenTopicsData, + Selector: `request.payload.forgottenTopicsData`, }, { - "name": "Rack ID", - "value": rackId, + Name: "Rack ID", + Value: rackId, + Selector: `request.payload.rackId`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -435,46 +472,50 @@ func representFetchResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) responses := "" - if payload["Responses"] != nil { - _responses, _ := json.Marshal(payload["Responses"].([]interface{})) + if payload["responses"] != nil { + _responses, _ := json.Marshal(payload["responses"].([]interface{})) responses = string(_responses) } throttleTimeMs := "" - if payload["ThrottleTimeMs"] != nil { - throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64))) + if payload["throttleTimeMs"] != nil { + throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) } errorCode := "" - if payload["ErrorCode"] != nil { - errorCode = fmt.Sprintf("%d", int(payload["ErrorCode"].(float64))) + if payload["errorCode"] != nil { + errorCode = fmt.Sprintf("%d", int(payload["errorCode"].(float64))) } sessionId := "" - if payload["SessionId"] != nil { - sessionId = fmt.Sprintf("%d", int(payload["SessionId"].(float64))) + if payload["sessionId"] != nil { + sessionId = fmt.Sprintf("%d", int(payload["sessionId"].(float64))) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Throttle Time (ms)", - "value": throttleTimeMs, + Name: "Throttle Time (ms)", + Value: throttleTimeMs, + Selector: `response.payload.throttleTimeMs`, }, { - "name": "Error Code", - "value": errorCode, + Name: "Error Code", + Value: errorCode, + Selector: `response.payload.errorCode`, }, { - "name": "Session ID", - "value": sessionId, + Name: "Session ID", + Value: sessionId, + Selector: `response.payload.sessionId`, }, { - "name": "Responses", - "value": responses, + Name: "Responses", + Value: responses, + Selector: `response.payload.responses`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -485,26 +526,28 @@ func representListOffsetsRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) topics := "" - if payload["Topics"] != nil { - _topics, _ := json.Marshal(payload["Topics"].([]interface{})) + if payload["topics"] != nil { + _topics, _ := json.Marshal(payload["topics"].([]interface{})) topics = string(_topics) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Replica ID", - "value": fmt.Sprintf("%d", int(payload["ReplicaId"].(float64))), + Name: "Replica ID", + Value: fmt.Sprintf("%d", int(payload["replicaId"].(float64))), + Selector: `request.payload.replicaId`, }, { - "name": "Topics", - "value": topics, + Name: "Topics", + Value: topics, + Selector: `request.payload.topics`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -515,26 +558,28 @@ func representListOffsetsResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) - topics, _ := json.Marshal(payload["Topics"].([]interface{})) + payload := data["payload"].(map[string]interface{}) + topics, _ := json.Marshal(payload["topics"].([]interface{})) throttleTimeMs := "" - if payload["ThrottleTimeMs"] != nil { - throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64))) + if payload["throttleTimeMs"] != nil { + throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Throttle Time (ms)", - "value": throttleTimeMs, + Name: "Throttle Time (ms)", + Value: throttleTimeMs, + Selector: `response.payload.throttleTimeMs`, }, { - "name": "Topics", - "value": string(topics), + Name: "Topics", + Value: string(topics), + Selector: `response.payload.topics`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -545,30 +590,33 @@ func representCreateTopicsRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) - topics, _ := json.Marshal(payload["Topics"].([]interface{})) + payload := data["payload"].(map[string]interface{}) + topics, _ := json.Marshal(payload["topics"].([]interface{})) validateOnly := "" - if payload["ValidateOnly"] != nil { - validateOnly = strconv.FormatBool(payload["ValidateOnly"].(bool)) + if payload["validateOnly"] != nil { + validateOnly = strconv.FormatBool(payload["validateOnly"].(bool)) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Topics", - "value": string(topics), + Name: "Topics", + Value: string(topics), + Selector: `request.payload.topics`, }, { - "name": "Timeout (ms)", - "value": fmt.Sprintf("%d", int(payload["TimeoutMs"].(float64))), + Name: "Timeout (ms)", + Value: fmt.Sprintf("%d", int(payload["timeoutMs"].(float64))), + Selector: `request.payload.timeoutMs`, }, { - "name": "Validate Only", - "value": validateOnly, + Name: "Validate Only", + Value: validateOnly, + Selector: `request.payload.validateOnly`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -579,26 +627,28 @@ func representCreateTopicsResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) - topics, _ := json.Marshal(payload["Topics"].([]interface{})) + payload := data["payload"].(map[string]interface{}) + topics, _ := json.Marshal(payload["topics"].([]interface{})) throttleTimeMs := "" - if payload["ThrottleTimeMs"] != nil { - throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64))) + if payload["throttleTimeMs"] != nil { + throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Throttle Time (ms)", - "value": throttleTimeMs, + Name: "Throttle Time (ms)", + Value: throttleTimeMs, + Selector: `response.payload.throttleTimeMs`, }, { - "name": "Topics", - "value": string(topics), + Name: "Topics", + Value: string(topics), + Selector: `response.payload.topics`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -609,35 +659,38 @@ func representDeleteTopicsRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) topics := "" - if payload["Topics"] != nil { - x, _ := json.Marshal(payload["Topics"].([]interface{})) + if payload["topics"] != nil { + x, _ := json.Marshal(payload["topics"].([]interface{})) topics = string(x) } topicNames := "" - if payload["TopicNames"] != nil { - x, _ := json.Marshal(payload["TopicNames"].([]interface{})) + if payload["topicNames"] != nil { + x, _ := json.Marshal(payload["topicNames"].([]interface{})) topicNames = string(x) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "TopicNames", - "value": string(topicNames), + Name: "TopicNames", + Value: string(topicNames), + Selector: `request.payload.topicNames`, }, { - "name": "Topics", - "value": string(topics), + Name: "Topics", + Value: string(topics), + Selector: `request.payload.topics`, }, { - "name": "Timeout (ms)", - "value": fmt.Sprintf("%d", int(payload["TimeoutMs"].(float64))), + Name: "Timeout (ms)", + Value: fmt.Sprintf("%d", int(payload["timeoutMs"].(float64))), + Selector: `request.payload.timeoutMs`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -648,26 +701,28 @@ func representDeleteTopicsResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) - responses, _ := json.Marshal(payload["Responses"].([]interface{})) + payload := data["payload"].(map[string]interface{}) + responses, _ := json.Marshal(payload["responses"].([]interface{})) throttleTimeMs := "" - if payload["ThrottleTimeMs"] != nil { - throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64))) + if payload["throttleTimeMs"] != nil { + throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Throttle Time (ms)", - "value": throttleTimeMs, + Name: "Throttle Time (ms)", + Value: throttleTimeMs, + Selector: `response.payload.throttleTimeMs`, }, { - "name": "Responses", - "value": string(responses), + Name: "Responses", + Value: string(responses), + Selector: `response.payload.responses`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep diff --git a/tap/extensions/kafka/main.go b/tap/extensions/kafka/main.go index b677182d1..43af66140 100644 --- a/tap/extensions/kafka/main.go +++ b/tap/extensions/kafka/main.go @@ -15,6 +15,7 @@ var _protocol api.Protocol = api.Protocol{ Name: "kafka", LongName: "Apache Kafka Protocol", Abbreviation: "KAFKA", + Macro: "kafka", Version: "12", BackgroundColor: "#000000", ForegroundColor: "#ffffff", @@ -36,7 +37,7 @@ func (d dissecting) Register(extension *api.Extension) { } func (d dissecting) Ping() { - log.Printf("pong %s\n", _protocol.Name) + log.Printf("pong %s", _protocol.Name) } func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error { @@ -61,85 +62,79 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co } } -func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry { +func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry { request := item.Pair.Request.Payload.(map[string]interface{}) reqDetails := request["details"].(map[string]interface{}) - service := "kafka" - if resolvedDestination != "" { - service = resolvedDestination - } else if resolvedSource != "" { - service = resolvedSource - } - apiKey := ApiKey(reqDetails["ApiKey"].(float64)) + apiKey := ApiKey(reqDetails["apiKey"].(float64)) summary := "" switch apiKey { case Metadata: - _topics := reqDetails["Payload"].(map[string]interface{})["Topics"] + _topics := reqDetails["payload"].(map[string]interface{})["topics"] if _topics == nil { break } topics := _topics.([]interface{}) for _, topic := range topics { - summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Name"].(string)) + summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["name"].(string)) } if len(summary) > 0 { summary = summary[:len(summary)-2] } break case ApiVersions: - summary = reqDetails["ClientID"].(string) + summary = reqDetails["clientID"].(string) break case Produce: - _topics := reqDetails["Payload"].(map[string]interface{})["TopicData"] + _topics := reqDetails["payload"].(map[string]interface{})["topicData"] if _topics == nil { break } topics := _topics.([]interface{}) for _, topic := range topics { - summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Topic"].(string)) + summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["topic"].(string)) } if len(summary) > 0 { summary = summary[:len(summary)-2] } break case Fetch: - _topics := reqDetails["Payload"].(map[string]interface{})["Topics"] + _topics := reqDetails["payload"].(map[string]interface{})["topics"] if _topics == nil { break } topics := _topics.([]interface{}) for _, topic := range topics { - summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Topic"].(string)) + summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["topic"].(string)) } if len(summary) > 0 { summary = summary[:len(summary)-2] } break case ListOffsets: - _topics := reqDetails["Payload"].(map[string]interface{})["Topics"] + _topics := reqDetails["payload"].(map[string]interface{})["topics"] if _topics == nil { break } topics := _topics.([]interface{}) for _, topic := range topics { - summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Name"].(string)) + summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["name"].(string)) } if len(summary) > 0 { summary = summary[:len(summary)-2] } break case CreateTopics: - topics := reqDetails["Payload"].(map[string]interface{})["Topics"].([]interface{}) + topics := reqDetails["payload"].(map[string]interface{})["topics"].([]interface{}) for _, topic := range topics { - summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Name"].(string)) + summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["name"].(string)) } if len(summary) > 0 { summary = summary[:len(summary)-2] } break case DeleteTopics: - topicNames := reqDetails["TopicNames"].([]string) + topicNames := reqDetails["topicNames"].([]string) for _, name := range topicNames { summary += fmt.Sprintf("%s, ", name) } @@ -148,53 +143,46 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve request["url"] = summary elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds() - entryBytes, _ := json.Marshal(item.Pair) + if elapsedTime < 0 { + elapsedTime = 0 + } return &api.MizuEntry{ - ProtocolName: _protocol.Name, - ProtocolLongName: _protocol.LongName, - ProtocolAbbreviation: _protocol.Abbreviation, - ProtocolVersion: _protocol.Version, - ProtocolBackgroundColor: _protocol.BackgroundColor, - ProtocolForegroundColor: _protocol.ForegroundColor, - ProtocolFontSize: _protocol.FontSize, - ProtocolReferenceLink: _protocol.ReferenceLink, - EntryId: entryId, - Entry: string(entryBytes), - Url: fmt.Sprintf("%s%s", service, summary), - Method: apiNames[apiKey], - Status: 0, - RequestSenderIp: item.ConnectionInfo.ClientIP, - Service: service, - Timestamp: item.Timestamp, - ElapsedTime: elapsedTime, - Path: summary, - ResolvedSource: resolvedSource, - ResolvedDestination: resolvedDestination, - SourceIp: item.ConnectionInfo.ClientIP, - DestinationIp: item.ConnectionInfo.ServerIP, - SourcePort: item.ConnectionInfo.ClientPort, - DestinationPort: item.ConnectionInfo.ServerPort, - IsOutgoing: item.ConnectionInfo.IsOutgoing, + Protocol: _protocol, + Source: &api.TCP{ + Name: resolvedSource, + IP: item.ConnectionInfo.ClientIP, + Port: item.ConnectionInfo.ClientPort, + }, + Destination: &api.TCP{ + Name: resolvedDestination, + IP: item.ConnectionInfo.ServerIP, + Port: item.ConnectionInfo.ServerPort, + }, + Outgoing: item.ConnectionInfo.IsOutgoing, + Request: reqDetails, + Response: item.Pair.Response.Payload.(map[string]interface{})["details"].(map[string]interface{}), + Method: apiNames[apiKey], + Status: 0, + Timestamp: item.Timestamp, + StartTime: item.Pair.Request.CaptureTime, + ElapsedTime: elapsedTime, + Summary: summary, + IsOutgoing: item.ConnectionInfo.IsOutgoing, } } func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { return &api.BaseEntryDetails{ - Id: entry.EntryId, - Protocol: _protocol, - Url: entry.Url, - RequestSenderIp: entry.RequestSenderIp, - Service: entry.Service, - Summary: entry.Path, - StatusCode: entry.Status, - Method: entry.Method, - Timestamp: entry.Timestamp, - SourceIp: entry.SourceIp, - DestinationIp: entry.DestinationIp, - SourcePort: entry.SourcePort, - DestinationPort: entry.DestinationPort, - IsOutgoing: entry.IsOutgoing, - Latency: entry.ElapsedTime, + Id: entry.Id, + Protocol: _protocol, + Summary: entry.Summary, + StatusCode: entry.Status, + Method: entry.Method, + Timestamp: entry.Timestamp, + Source: entry.Source, + Destination: entry.Destination, + IsOutgoing: entry.IsOutgoing, + Latency: entry.ElapsedTime, Rules: api.ApplicableRules{ Latency: 0, Status: false, @@ -202,49 +190,42 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { } } -func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) { - p = _protocol +func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) { bodySize = 0 - var root map[string]interface{} - json.Unmarshal([]byte(entry.Entry), &root) representation := make(map[string]interface{}, 0) - request := root["request"].(map[string]interface{})["payload"].(map[string]interface{}) - response := root["response"].(map[string]interface{})["payload"].(map[string]interface{}) - reqDetails := request["details"].(map[string]interface{}) - resDetails := response["details"].(map[string]interface{}) - apiKey := ApiKey(reqDetails["ApiKey"].(float64)) + apiKey := ApiKey(request["apiKey"].(float64)) var repRequest []interface{} var repResponse []interface{} switch apiKey { case Metadata: - repRequest = representMetadataRequest(reqDetails) - repResponse = representMetadataResponse(resDetails) + repRequest = representMetadataRequest(request) + repResponse = representMetadataResponse(response) break case ApiVersions: - repRequest = representApiVersionsRequest(reqDetails) - repResponse = representApiVersionsResponse(resDetails) + repRequest = representApiVersionsRequest(request) + repResponse = representApiVersionsResponse(response) break case Produce: - repRequest = representProduceRequest(reqDetails) - repResponse = representProduceResponse(resDetails) + repRequest = representProduceRequest(request) + repResponse = representProduceResponse(response) break case Fetch: - repRequest = representFetchRequest(reqDetails) - repResponse = representFetchResponse(resDetails) + repRequest = representFetchRequest(request) + repResponse = representFetchResponse(response) break case ListOffsets: - repRequest = representListOffsetsRequest(reqDetails) - repResponse = representListOffsetsResponse(resDetails) + repRequest = representListOffsetsRequest(request) + repResponse = representListOffsetsResponse(response) break case CreateTopics: - repRequest = representCreateTopicsRequest(reqDetails) - repResponse = representCreateTopicsResponse(resDetails) + repRequest = representCreateTopicsRequest(request) + repResponse = representCreateTopicsResponse(response) break case DeleteTopics: - repRequest = representDeleteTopicsRequest(reqDetails) - repResponse = representDeleteTopicsResponse(resDetails) + repRequest = representDeleteTopicsRequest(request) + repResponse = representDeleteTopicsResponse(response) break } @@ -254,4 +235,10 @@ func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []by return } +func (d dissecting) Macros() map[string]string { + return map[string]string{ + `kafka`: fmt.Sprintf(`proto.name == "%s"`, _protocol.Name), + } +} + var Dissector dissecting diff --git a/tap/extensions/kafka/request.go b/tap/extensions/kafka/request.go index b7ac67d7d..263755892 100644 --- a/tap/extensions/kafka/request.go +++ b/tap/extensions/kafka/request.go @@ -10,13 +10,13 @@ import ( ) type Request struct { - Size int32 - ApiKey ApiKey - ApiVersion int16 - CorrelationID int32 - ClientID string - Payload interface{} - CaptureTime time.Time + Size int32 `json:"size"` + ApiKey ApiKey `json:"apiKey"` + ApiVersion int16 `json:"apiVersion"` + CorrelationID int32 `json:"correlationID"` + ClientID string `json:"clientID"` + Payload interface{} `json:"payload"` + CaptureTime time.Time `json:"captureTime"` } func ReadRequest(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer) (apiKey ApiKey, apiVersion int16, err error) { diff --git a/tap/extensions/kafka/response.go b/tap/extensions/kafka/response.go index 574efa8a2..5441e784d 100644 --- a/tap/extensions/kafka/response.go +++ b/tap/extensions/kafka/response.go @@ -10,10 +10,10 @@ import ( ) type Response struct { - Size int32 - CorrelationID int32 - Payload interface{} - CaptureTime time.Time + Size int32 `json:"size"` + CorrelationID int32 `json:"correlationID"` + Payload interface{} `json:"payload"` + CaptureTime time.Time `json:"captureTime"` } func ReadResponse(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter) (err error) { diff --git a/tap/extensions/kafka/structs.go b/tap/extensions/kafka/structs.go index d9aa5c1cb..67064d85e 100644 --- a/tap/extensions/kafka/structs.go +++ b/tap/extensions/kafka/structs.go @@ -26,38 +26,38 @@ func (acks RequiredAcks) String() string { } type UUID struct { - TimeLow int32 - TimeMid int16 - TimeHiAndVersion int16 - ClockSeq int16 - NodePart1 int32 - NodePart22 int16 + TimeLow int32 `json:"timeLow"` + TimeMid int16 `json:"timeMid"` + TimeHiAndVersion int16 `json:"timeHiAndVersion"` + ClockSeq int16 `json:"clockSeq"` + NodePart1 int32 `json:"nodePart1"` + NodePart22 int16 `json:"nodePart22"` } // Metadata Request (Version: 0) type MetadataRequestTopicV0 struct { - Name string + Name string `json:"name"` } type MetadataRequestV0 struct { - Topics []MetadataRequestTopicV0 + Topics []MetadataRequestTopicV0 `json:"topics"` } // Metadata Request (Version: 4) type MetadataRequestV4 struct { - Topics []MetadataRequestTopicV0 - AllowAutoTopicCreation bool + Topics []MetadataRequestTopicV0 `json:"topics"` + AllowAutoTopicCreation bool `json:"allowAutoTopicCreation"` } // Metadata Request (Version: 8) type MetadataRequestV8 struct { - Topics []MetadataRequestTopicV0 - AllowAutoTopicCreation bool - IncludeClusterAuthorizedOperations bool - IncludeTopicAuthorizedOperations bool + Topics []MetadataRequestTopicV0 `json:"topics"` + AllowAutoTopicCreation bool `json:"allowAutoTopicCreation"` + IncludeClusterAuthorizedOperations bool `json:"includeClusterAuthorizedOperations"` + IncludeTopicAuthorizedOperations bool `json:"includeTopicAuthorizedOperations"` } // Metadata Request (Version: 10) @@ -68,188 +68,188 @@ type MetadataRequestTopicV10 struct { } type MetadataRequestV10 struct { - Topics []MetadataRequestTopicV10 - AllowAutoTopicCreation bool - IncludeClusterAuthorizedOperations bool - IncludeTopicAuthorizedOperations bool + Topics []MetadataRequestTopicV10 `json:"topics"` + AllowAutoTopicCreation bool `json:"allowAutoTopicCreation"` + IncludeClusterAuthorizedOperations bool `json:"includeClusterAuthorizedOperations"` + IncludeTopicAuthorizedOperations bool `json:"includeTopicAuthorizedOperations"` } // Metadata Request (Version: 11) type MetadataRequestV11 struct { - Topics []MetadataRequestTopicV10 - AllowAutoTopicCreation bool - IncludeTopicAuthorizedOperations bool + Topics []MetadataRequestTopicV10 `json:"topics"` + AllowAutoTopicCreation bool `json:"allowAutoTopicCreation"` + IncludeTopicAuthorizedOperations bool `json:"includeTopicAuthorizedOperations"` } // Metadata Response (Version: 0) type BrokerV0 struct { - NodeId int32 - Host string - Port int32 + NodeId int32 `json:"nodeId"` + Host string `json:"host"` + Port int32 `json:"port"` } type PartitionsV0 struct { - ErrorCode int16 - PartitionIndex int32 - LeaderId int32 - ReplicaNodes int32 - IsrNodes int32 + ErrorCode int16 `json:"errorCode"` + PartitionIndex int32 `json:"partitionIndex"` + LeaderId int32 `json:"leaderId"` + ReplicaNodes int32 `json:"replicaNodes"` + IsrNodes int32 `json:"isrNodes"` } type TopicV0 struct { - ErrorCode int16 - Name string - Partitions []PartitionsV0 + ErrorCode int16 `json:"errorCode"` + Name string `json:"name"` + Partitions []PartitionsV0 `json:"partitions"` } type MetadataResponseV0 struct { - Brokers []BrokerV0 - Topics []TopicV0 + Brokers []BrokerV0 `json:"brokers"` + Topics []TopicV0 `json:"topics"` } // Metadata Response (Version: 1) type BrokerV1 struct { - NodeId int32 - Host string - Port int32 - Rack string + NodeId int32 `json:"nodeId"` + Host string `json:"host"` + Port int32 `json:"port"` + Rack string `json:"rack"` } type TopicV1 struct { - ErrorCode int16 - Name string - IsInternal bool - Partitions []PartitionsV0 + ErrorCode int16 `json:"errorCode"` + Name string `json:"name"` + IsInternal bool `json:"isInternal"` + Partitions []PartitionsV0 `json:"partitions"` } type MetadataResponseV1 struct { - Brokers []BrokerV1 - ControllerID int32 - Topics []TopicV1 + Brokers []BrokerV1 `json:"brokers"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV1 `json:"topics"` } // Metadata Response (Version: 2) type MetadataResponseV2 struct { - Brokers []BrokerV1 - ClusterID string - ControllerID int32 - Topics []TopicV1 + Brokers []BrokerV1 `json:"brokers"` + ClusterID string `json:"clusterID"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV1 `json:"topics"` } // Metadata Response (Version: 3) type MetadataResponseV3 struct { - ThrottleTimeMs int32 - Brokers []BrokerV1 - ClusterID string - ControllerID int32 - Topics []TopicV1 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Brokers []BrokerV1 `json:"brokers"` + ClusterID string `json:"clusterID"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV1 `json:"topics"` } // Metadata Response (Version: 5) type PartitionsV5 struct { - ErrorCode int16 - PartitionIndex int32 - LeaderId int32 - ReplicaNodes int32 - IsrNodes int32 - OfflineReplicas int32 + ErrorCode int16 `json:"errorCode"` + PartitionIndex int32 `json:"partitionIndex"` + LeaderId int32 `json:"leaderId"` + ReplicaNodes int32 `json:"replicaNodes"` + IsrNodes int32 `json:"isrNodes"` + OfflineReplicas int32 `json:"offlineReplicas"` } type TopicV5 struct { - ErrorCode int16 - Name string - IsInternal bool - Partitions []PartitionsV5 + ErrorCode int16 `json:"errorCode"` + Name string `json:"name"` + IsInternal bool `json:"isInternal"` + Partitions []PartitionsV5 `json:"partitions"` } type MetadataResponseV5 struct { - ThrottleTimeMs int32 - Brokers []BrokerV1 - ClusterID string - ControllerID int32 - Topics []TopicV5 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Brokers []BrokerV1 `json:"brokers"` + ClusterID string `json:"clusterID"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV5 `json:"topics"` } // Metadata Response (Version: 7) type PartitionsV7 struct { - ErrorCode int16 - PartitionIndex int32 - LeaderId int32 - LeaderEpoch int32 - ReplicaNodes int32 - IsrNodes int32 - OfflineReplicas int32 + ErrorCode int16 `json:"errorCode"` + PartitionIndex int32 `json:"partitionIndex"` + LeaderId int32 `json:"leaderId"` + LeaderEpoch int32 `json:"leaderEpoch"` + ReplicaNodes int32 `json:"replicaNodes"` + IsrNodes int32 `json:"isrNodes"` + OfflineReplicas int32 `json:"offlineReplicas"` } type TopicV7 struct { - ErrorCode int16 - Name string - IsInternal bool - Partitions []PartitionsV7 + ErrorCode int16 `json:"errorCode"` + Name string `json:"name"` + IsInternal bool `json:"isInternal"` + Partitions []PartitionsV7 `json:"partitions"` } type MetadataResponseV7 struct { - ThrottleTimeMs int32 - Brokers []BrokerV1 - ClusterID string - ControllerID int32 - Topics []TopicV7 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Brokers []BrokerV1 `json:"brokers"` + ClusterID string `json:"clusterID"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV7 `json:"topics"` } // Metadata Response (Version: 8) type TopicV8 struct { - ErrorCode int16 - Name string - IsInternal bool - Partitions []PartitionsV7 - TopicAuthorizedOperations int32 + ErrorCode int16 `json:"errorCode"` + Name string `json:"name"` + IsInternal bool `json:"isInternal"` + Partitions []PartitionsV7 `json:"partitions"` + TopicAuthorizedOperations int32 `json:"topicAuthorizedOperations"` } type MetadataResponseV8 struct { - ThrottleTimeMs int32 - Brokers []BrokerV1 - ClusterID string - ControllerID int32 - Topics []TopicV8 - ClusterAuthorizedOperations int32 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Brokers []BrokerV1 `json:"brokers"` + ClusterID string `json:"clusterID"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV8 `json:"topics"` + ClusterAuthorizedOperations int32 `json:"clusterAuthorizedOperations"` } // Metadata Response (Version: 10) type TopicV10 struct { - ErrorCode int16 - Name string - TopicID UUID - IsInternal bool - Partitions []PartitionsV7 - TopicAuthorizedOperations int32 + ErrorCode int16 `json:"errorCode"` + Name string `json:"name"` + TopicID UUID `json:"topicID"` + IsInternal bool `json:"isInternal"` + Partitions []PartitionsV7 `json:"partitions"` + TopicAuthorizedOperations int32 `json:"topicAuthorizedOperations"` } type MetadataResponseV10 struct { - ThrottleTimeMs int32 - Brokers []BrokerV1 - ClusterID string - ControllerID int32 - Topics []TopicV10 - ClusterAuthorizedOperations int32 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Brokers []BrokerV1 `json:"brokers"` + ClusterID string `json:"clusterID"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV10 `json:"topics"` + ClusterAuthorizedOperations int32 `json:"clusterAuthorizedOperations"` } // Metadata Response (Version: 11) type MetadataResponseV11 struct { - ThrottleTimeMs int32 - Brokers []BrokerV1 - ClusterID string - ControllerID int32 - Topics []TopicV10 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Brokers []BrokerV1 `json:"brokers"` + ClusterID string `json:"clusterID"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV10 `json:"topics"` } // ApiVersions Request (Version: 0) @@ -259,742 +259,742 @@ type ApiVersionsRequestV0 struct{} // ApiVersions Request (Version: 3) type ApiVersionsRequestV3 struct { - ClientSoftwareName string - ClientSoftwareVersion string + ClientSoftwareName string `json:"clientSoftwareName"` + ClientSoftwareVersion string `json:"clientSoftwareVersion"` } // ApiVersions Response (Version: 0) type ApiVersionsResponseApiKey struct { - ApiKey int16 - MinVersion int16 - MaxVersion int16 + ApiKey int16 `json:"apiKey"` + MinVersion int16 `json:"minVersion"` + MaxVersion int16 `json:"maxVersion"` } type ApiVersionsResponseV0 struct { - ErrorCode int16 - ApiKeys []ApiVersionsResponseApiKey + ErrorCode int16 `json:"errorCode"` + ApiKeys []ApiVersionsResponseApiKey `json:"apiKeys"` } // ApiVersions Response (Version: 1) type ApiVersionsResponseV1 struct { - ErrorCode int16 - ApiKeys []ApiVersionsResponseApiKey // FIXME: `confluent-kafka-python` causes memory leak - ThrottleTimeMs int32 + ErrorCode int16 `json:"errorCode"` + ApiKeys []ApiVersionsResponseApiKey `json:"apiKeys"` // FIXME: `confluent-kafka-python` causes memory leak + ThrottleTimeMs int32 `json:"throttleTimeMs"` } // Produce Request (Version: 0) // Message is a kafka message type type MessageV0 struct { - Codec int8 // codec used to compress the message contents - CompressionLevel int // compression level - LogAppendTime bool // the used timestamp is LogAppendTime - Key []byte // the message key, may be nil - Value []byte // the message contents - Set *MessageSet // the message set a message might wrap - Version int8 // v1 requires Kafka 0.10 - Timestamp time.Time // the timestamp of the message (version 1+ only) + Codec int8 `json:"codec"` // codec used to compress the message contents + CompressionLevel int `json:"compressionLevel"` // compression level + LogAppendTime bool `json:"logAppendTime"` // the used timestamp is LogAppendTime + Key []byte `json:"key"` // the message key, may be nil + Value []byte `json:"value"` // the message contents + Set *MessageSet `json:"set"` // the message set a message might wrap + Version int8 `json:"version"` // v1 requires Kafka 0.10 + Timestamp time.Time `json:"timestamp"` // the timestamp of the message (version 1+ only) compressedSize int // used for computing the compression ratio metrics } // MessageBlock represents a part of request with message type MessageBlock struct { - Offset int64 - Msg *MessageV0 + Offset int64 `json:"offset"` + Msg *MessageV0 `json:"msg"` } // MessageSet is a replacement for RecordBatch in older versions type MessageSet struct { - PartialTrailingMessage bool // whether the set on the wire contained an incomplete trailing MessageBlock - OverflowMessage bool // whether the set on the wire contained an overflow message - Messages []*MessageBlock + PartialTrailingMessage bool `json:"partialTrailingMessage"` // whether the set on the wire contained an incomplete trailing MessageBlock + OverflowMessage bool `json:"overflowMessage"` // whether the set on the wire contained an overflow message + Messages []*MessageBlock `json:"messages"` } type RecordHeader struct { - HeaderKeyLength int8 - HeaderKey string - HeaderValueLength int8 - Value string + HeaderKeyLength int8 `json:"headerKeyLength"` + HeaderKey string `json:"headerKey"` + HeaderValueLength int8 `json:"headerValueLength"` + Value string `json:"value"` } // Record is kafka record type type RecordV0 struct { - Unknown int8 - Attributes int8 - TimestampDelta int8 - OffsetDelta int8 - KeyLength int8 - Key string - ValueLen int8 - Value string - Headers []RecordHeader + Unknown int8 `json:"unknown"` + Attributes int8 `json:"attributes"` + TimestampDelta int8 `json:"timestampDelta"` + OffsetDelta int8 `json:"offsetDelta"` + KeyLength int8 `json:"keyLength"` + Key string `json:"key"` + ValueLen int8 `json:"valueLen"` + Value string `json:"value"` + Headers []RecordHeader `json:"headers"` } // RecordBatch are records from one kafka request type RecordBatch struct { - BaseOffset int64 - BatchLength int32 - PartitionLeaderEpoch int32 - Magic int8 - Crc int32 - Attributes int16 - LastOffsetDelta int32 - FirstTimestamp int64 - MaxTimestamp int64 - ProducerId int64 - ProducerEpoch int16 - BaseSequence int32 - Record []RecordV0 + BaseOffset int64 `json:"baseOffset"` + BatchLength int32 `json:"batchLength"` + PartitionLeaderEpoch int32 `json:"partitionLeaderEpoch"` + Magic int8 `json:"magic"` + Crc int32 `json:"crc"` + Attributes int16 `json:"attributes"` + LastOffsetDelta int32 `json:"lastOffsetDelta"` + FirstTimestamp int64 `json:"firstTimestamp"` + MaxTimestamp int64 `json:"maxTimestamp"` + ProducerId int64 `json:"producerId"` + ProducerEpoch int16 `json:"producerEpoch"` + BaseSequence int32 `json:"baseSequence"` + Record []RecordV0 `json:"record"` } type Records struct { - RecordBatch RecordBatch + RecordBatch RecordBatch `json:"recordBatch"` // TODO: Implement `MessageSet` // MessageSet MessageSet } type PartitionData struct { - Index int32 - Unknown int32 - Records Records + Index int32 `json:"index"` + Unknown int32 `json:"unknown"` + Records Records `json:"records"` } type Partitions struct { - Length int32 - PartitionData PartitionData + Length int32 `json:"length"` + PartitionData PartitionData `json:"partitionData"` } type TopicData struct { - Topic string - Partitions Partitions + Topic string `json:"topic"` + Partitions Partitions `json:"partitions"` } type ProduceRequestV0 struct { - RequiredAcks RequiredAcks - Timeout int32 - TopicData []TopicData + RequiredAcks RequiredAcks `json:"requiredAcks"` + Timeout int32 `json:"timeout"` + TopicData []TopicData `json:"topicData"` } // Produce Request (Version: 3) type ProduceRequestV3 struct { - TransactionalID string - RequiredAcks RequiredAcks - Timeout int32 - TopicData []TopicData + TransactionalID string `json:"transactionalID"` + RequiredAcks RequiredAcks `json:"requiredAcks"` + Timeout int32 `json:"timeout"` + TopicData []TopicData `json:"topicData"` } // Produce Response (Version: 0) type PartitionResponseV0 struct { - Index int32 - ErrorCode int16 - BaseOffset int64 + Index int32 `json:"index"` + ErrorCode int16 `json:"errorCode"` + BaseOffset int64 `json:"baseOffset"` } type ResponseV0 struct { - Name string - PartitionResponses []PartitionResponseV0 + Name string `json:"name"` + PartitionResponses []PartitionResponseV0 `json:"partitionResponses"` } type ProduceResponseV0 struct { - Responses []ResponseV0 + Responses []ResponseV0 `json:"responses"` } // Produce Response (Version: 1) type ProduceResponseV1 struct { - Responses []ResponseV0 - ThrottleTimeMs int32 + Responses []ResponseV0 `json:"responses"` + ThrottleTimeMs int32 `json:"throttleTimeMs"` } // Produce Response (Version: 2) type PartitionResponseV2 struct { - Index int32 - ErrorCode int16 - BaseOffset int64 - LogAppendTimeMs int64 + Index int32 `json:"index"` + ErrorCode int16 `json:"errorCode"` + BaseOffset int64 `json:"baseOffset"` + LogAppendTimeMs int64 `json:"logAppendTimeMs"` } type ResponseV2 struct { - Name string - PartitionResponses []PartitionResponseV2 + Name string `json:"name"` + PartitionResponses []PartitionResponseV2 `json:"partitionResponses"` } type ProduceResponseV2 struct { - Responses []ResponseV2 - ThrottleTimeMs int32 + Responses []ResponseV2 `json:"responses"` + ThrottleTimeMs int32 `json:"throttleTimeMs"` } // Produce Response (Version: 5) type PartitionResponseV5 struct { - Index int32 - ErrorCode int16 - BaseOffset int64 - LogAppendTimeMs int64 - LogStartOffset int64 + Index int32 `json:"index"` + ErrorCode int16 `json:"errorCode"` + BaseOffset int64 `json:"baseOffset"` + LogAppendTimeMs int64 `json:"logAppendTimeMs"` + LogStartOffset int64 `json:"logStartOffset"` } type ResponseV5 struct { - Name string - PartitionResponses []PartitionResponseV5 + Name string `json:"name"` + PartitionResponses []PartitionResponseV5 `json:"partitionResponses"` } type ProduceResponseV5 struct { - Responses []ResponseV5 - ThrottleTimeMs int32 + Responses []ResponseV5 `json:"responses"` + ThrottleTimeMs int32 `json:"throttleTimeMs"` } // Produce Response (Version: 8) type RecordErrors struct { - BatchIndex int32 - BatchIndexErrorMessage string + BatchIndex int32 `json:"batchIndex"` + BatchIndexErrorMessage string `json:"batchIndexErrorMessage"` } type PartitionResponseV8 struct { - Index int32 - ErrorCode int16 - BaseOffset int64 - LogAppendTimeMs int64 - LogStartOffset int64 - RecordErrors RecordErrors - ErrorMessage string + Index int32 `json:"index"` + ErrorCode int16 `json:"errorCode"` + BaseOffset int64 `json:"baseOffset"` + LogAppendTimeMs int64 `json:"logAppendTimeMs"` + LogStartOffset int64 `json:"logStartOffset"` + RecordErrors RecordErrors `json:"recordErrors"` + ErrorMessage string `json:"errorMessage"` } type ResponseV8 struct { - Name string - PartitionResponses []PartitionResponseV8 + Name string `json:"name"` + PartitionResponses []PartitionResponseV8 `json:"partitionResponses"` } type ProduceResponseV8 struct { - Responses []ResponseV8 - ThrottleTimeMs int32 + Responses []ResponseV8 `json:"responses"` + ThrottleTimeMs int32 `json:"throttleTimeMs"` } // Fetch Request (Version: 0) type FetchPartitionV0 struct { - Partition int32 - FetchOffset int64 - PartitionMaxBytes int32 + Partition int32 `json:"partition"` + FetchOffset int64 `json:"fetchOffset"` + PartitionMaxBytes int32 `json:"partitionMaxBytes"` } type FetchTopicV0 struct { - Topic string - Partitions []FetchPartitionV0 + Topic string `json:"topic"` + Partitions []FetchPartitionV0 `json:"partitions"` } type FetchRequestV0 struct { - ReplicaId int32 - MaxWaitMs int32 - MinBytes int32 - Topics []FetchTopicV0 + ReplicaId int32 `json:"replicaId"` + MaxWaitMs int32 `json:"maxWaitMs"` + MinBytes int32 `json:"minBytes"` + Topics []FetchTopicV0 `json:"topics"` } // Fetch Request (Version: 3) type FetchRequestV3 struct { - ReplicaId int32 - MaxWaitMs int32 - MinBytes int32 - MaxBytes int32 - Topics []FetchTopicV0 + ReplicaId int32 `json:"replicaId"` + MaxWaitMs int32 `json:"maxWaitMs"` + MinBytes int32 `json:"minBytes"` + MaxBytes int32 `json:"maxBytes"` + Topics []FetchTopicV0 `json:"topics"` } // Fetch Request (Version: 4) type FetchRequestV4 struct { - ReplicaId int32 - MaxWaitMs int32 - MinBytes int32 - MaxBytes int32 - IsolationLevel int8 - Topics []FetchTopicV0 + ReplicaId int32 `json:"replicaId"` + MaxWaitMs int32 `json:"maxWaitMs"` + MinBytes int32 `json:"minBytes"` + MaxBytes int32 `json:"maxBytes"` + IsolationLevel int8 `json:"isolationLevel"` + Topics []FetchTopicV0 `json:"topics"` } // Fetch Request (Version: 5) type FetchPartitionV5 struct { - Partition int32 - FetchOffset int64 - LogStartOffset int64 - PartitionMaxBytes int32 + Partition int32 `json:"partition"` + FetchOffset int64 `json:"fetchOffset"` + LogStartOffset int64 `json:"logStartOffset"` + PartitionMaxBytes int32 `json:"partitionMaxBytes"` } type FetchTopicV5 struct { - Topic string - Partitions []FetchPartitionV5 + Topic string `json:"topic"` + Partitions []FetchPartitionV5 `json:"partitions"` } type FetchRequestV5 struct { - ReplicaId int32 - MaxWaitMs int32 - MinBytes int32 - MaxBytes int32 - IsolationLevel int8 - Topics []FetchTopicV5 + ReplicaId int32 `json:"replicaId"` + MaxWaitMs int32 `json:"maxWaitMs"` + MinBytes int32 `json:"minBytes"` + MaxBytes int32 `json:"maxBytes"` + IsolationLevel int8 `json:"isolationLevel"` + Topics []FetchTopicV5 `json:"topics"` } // Fetch Request (Version: 7) type ForgottenTopicsDataV7 struct { - Topic string - Partitions []int32 + Topic string `json:"topic"` + Partitions []int32 `json:"partitions"` } type FetchRequestV7 struct { - ReplicaId int32 - MaxWaitMs int32 - MinBytes int32 - MaxBytes int32 - IsolationLevel int8 - SessionId int32 - SessionEpoch int32 - Topics []FetchTopicV5 - ForgottenTopicsData ForgottenTopicsDataV7 + ReplicaId int32 `json:"replicaId"` + MaxWaitMs int32 `json:"maxWaitMs"` + MinBytes int32 `json:"minBytes"` + MaxBytes int32 `json:"maxBytes"` + IsolationLevel int8 `json:"isolationLevel"` + SessionId int32 `json:"sessionId"` + SessionEpoch int32 `json:"sessionEpoch"` + Topics []FetchTopicV5 `json:"topics"` + ForgottenTopicsData ForgottenTopicsDataV7 `json:"forgottenTopicsData"` } // Fetch Request (Version: 9) type FetchPartitionV9 struct { - Partition int32 - CurrentLeaderEpoch int32 - FetchOffset int64 - LogStartOffset int64 - PartitionMaxBytes int32 + Partition int32 `json:"partition"` + CurrentLeaderEpoch int32 `json:"currentLeaderEpoch"` + FetchOffset int64 `json:"fetchOffset"` + LogStartOffset int64 `json:"logStartOffset"` + PartitionMaxBytes int32 `json:"partitionMaxBytes"` } type FetchTopicV9 struct { - Topic string - Partitions []FetchPartitionV9 + Topic string `json:"topic"` + Partitions []FetchPartitionV9 `json:"partitions"` } type FetchRequestV9 struct { - ReplicaId int32 - MaxWaitMs int32 - MinBytes int32 - MaxBytes int32 - IsolationLevel int8 - SessionId int32 - SessionEpoch int32 - Topics []FetchTopicV9 - ForgottenTopicsData ForgottenTopicsDataV7 + ReplicaId int32 `json:"replicaId"` + MaxWaitMs int32 `json:"maxWaitMs"` + MinBytes int32 `json:"minBytes"` + MaxBytes int32 `json:"maxBytes"` + IsolationLevel int8 `json:"isolationLevel"` + SessionId int32 `json:"sessionId"` + SessionEpoch int32 `json:"sessionEpoch"` + Topics []FetchTopicV9 `json:"topics"` + ForgottenTopicsData ForgottenTopicsDataV7 `json:"forgottenTopicsData"` } // Fetch Request (Version: 11) type FetchRequestV11 struct { - ReplicaId int32 - MaxWaitMs int32 - MinBytes int32 - MaxBytes int32 - IsolationLevel int8 - SessionId int32 - SessionEpoch int32 - Topics []FetchTopicV9 - ForgottenTopicsData ForgottenTopicsDataV7 - RackId string + ReplicaId int32 `json:"replicaId"` + MaxWaitMs int32 `json:"maxWaitMs"` + MinBytes int32 `json:"minBytes"` + MaxBytes int32 `json:"maxBytes"` + IsolationLevel int8 `json:"isolationLevel"` + SessionId int32 `json:"sessionId"` + SessionEpoch int32 `json:"sessionEpoch"` + Topics []FetchTopicV9 `json:"topics"` + ForgottenTopicsData ForgottenTopicsDataV7 `json:"forgottenTopicsData"` + RackId string `json:"rackId"` } // Fetch Response (Version: 0) type PartitionResponseFetchV0 struct { - Partition int32 - ErrorCode int16 - HighWatermark int64 - RecordSet Records + Partition int32 `json:"partition"` + ErrorCode int16 `json:"errorCode"` + HighWatermark int64 `json:"highWatermark"` + RecordSet Records `json:"recordSet"` } type ResponseFetchV0 struct { - Topic string - PartitionResponses []PartitionResponseFetchV0 + Topic string `json:"topic"` + PartitionResponses []PartitionResponseFetchV0 `json:"partitionResponses"` } type FetchResponseV0 struct { - Responses []ResponseFetchV0 + Responses []ResponseFetchV0 `json:"responses"` } // Fetch Response (Version: 1) type FetchResponseV1 struct { - ThrottleTimeMs int32 - Responses []ResponseFetchV0 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Responses []ResponseFetchV0 `json:"responses"` } // Fetch Response (Version: 4) type AbortedTransactionsV4 struct { - ProducerId int32 - FirstOffset int32 + ProducerId int32 `json:"producerId"` + FirstOffset int32 `json:"firstOffset"` } type PartitionResponseFetchV4 struct { - Partition int32 - ErrorCode int16 - HighWatermark int64 - LastStableOffset int64 - AbortedTransactions AbortedTransactionsV4 - RecordSet Records + Partition int32 `json:"partition"` + ErrorCode int16 `json:"errorCode"` + HighWatermark int64 `json:"highWatermark"` + LastStableOffset int64 `json:"lastStableOffset"` + AbortedTransactions AbortedTransactionsV4 `json:"abortedTransactions"` + RecordSet Records `json:"recordSet"` } type ResponseFetchV4 struct { - Topic string - PartitionResponses []PartitionResponseFetchV4 + Topic string `json:"topic"` + PartitionResponses []PartitionResponseFetchV4 `json:"partitionResponses"` } type FetchResponseV4 struct { - ThrottleTimeMs int32 - Responses []ResponseFetchV4 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Responses []ResponseFetchV4 `json:"responses"` } // Fetch Response (Version: 5) type PartitionResponseFetchV5 struct { - Partition int32 - ErrorCode int16 - HighWatermark int64 - LastStableOffset int64 - LogStartOffset int64 - AbortedTransactions AbortedTransactionsV4 - RecordSet Records + Partition int32 `json:"partition"` + ErrorCode int16 `json:"errorCode"` + HighWatermark int64 `json:"highWatermark"` + LastStableOffset int64 `json:"lastStableOffset"` + LogStartOffset int64 `json:"logStartOffset"` + AbortedTransactions AbortedTransactionsV4 `json:"abortedTransactions"` + RecordSet Records `json:"recordSet"` } type ResponseFetchV5 struct { - Topic string - PartitionResponses []PartitionResponseFetchV5 + Topic string `json:"topic"` + PartitionResponses []PartitionResponseFetchV5 `json:"partitionResponses"` } type FetchResponseV5 struct { - ThrottleTimeMs int32 - Responses []ResponseFetchV5 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Responses []ResponseFetchV5 `json:"responses"` } // Fetch Response (Version: 7) type FetchResponseV7 struct { - ThrottleTimeMs int32 - ErrorCode int16 - SessionId int32 - Responses []ResponseFetchV5 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + ErrorCode int16 `json:"errorCode"` + SessionId int32 `json:"sessionId"` + Responses []ResponseFetchV5 `json:"responses"` } // Fetch Response (Version: 11) type PartitionResponseFetchV11 struct { - Partition int32 - ErrorCode int16 - HighWatermark int64 - LastStableOffset int64 - LogStartOffset int64 - AbortedTransactions AbortedTransactionsV4 - PreferredReadReplica int32 - RecordSet Records + Partition int32 `json:"partition"` + ErrorCode int16 `json:"errorCode"` + HighWatermark int64 `json:"highWatermark"` + LastStableOffset int64 `json:"lastStableOffset"` + LogStartOffset int64 `json:"logStartOffset"` + AbortedTransactions AbortedTransactionsV4 `json:"abortedTransactions"` + PreferredReadReplica int32 `json:"preferredReadReplica"` + RecordSet Records `json:"recordSet"` } type ResponseFetchV11 struct { - Topic string - PartitionResponses []PartitionResponseFetchV11 + Topic string `json:"topic"` + PartitionResponses []PartitionResponseFetchV11 `json:"partitionResponses"` } type FetchResponseV11 struct { - ThrottleTimeMs int32 - ErrorCode int16 - SessionId int32 - Responses []ResponseFetchV5 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + ErrorCode int16 `json:"errorCode"` + SessionId int32 `json:"sessionId"` + Responses []ResponseFetchV5 `json:"responses"` } // ListOffsets Request (Version: 0) type ListOffsetsRequestPartitionV0 struct { - PartitionIndex int32 - Timestamp int64 - MaxNumOffsets int32 + PartitionIndex int32 `json:"partitionIndex"` + Timestamp int64 `json:"timestamp"` + MaxNumOffsets int32 `json:"maxNumOffsets"` } type ListOffsetsRequestTopicV0 struct { - Name string - Partitions []ListOffsetsRequestPartitionV0 + Name string `json:"name"` + Partitions []ListOffsetsRequestPartitionV0 `json:"partitions"` } type ListOffsetsRequestV0 struct { - ReplicaId int32 - Topics []ListOffsetsRequestTopicV0 + ReplicaId int32 `json:"replicaId"` + Topics []ListOffsetsRequestTopicV0 `json:"topics"` } // ListOffsets Request (Version: 1) type ListOffsetsRequestPartitionV1 struct { - PartitionIndex int32 - Timestamp int64 + PartitionIndex int32 `json:"partitionIndex"` + Timestamp int64 `json:"timestamp"` } type ListOffsetsRequestTopicV1 struct { - Name string - Partitions []ListOffsetsRequestPartitionV1 + Name string `json:"name"` + Partitions []ListOffsetsRequestPartitionV1 `json:"partitions"` } type ListOffsetsRequestV1 struct { - ReplicaId int32 - Topics []ListOffsetsRequestTopicV1 + ReplicaId int32 `json:"replicaId"` + Topics []ListOffsetsRequestTopicV1 `json:"topics"` } // ListOffsets Request (Version: 2) type ListOffsetsRequestV2 struct { - ReplicaId int32 - IsolationLevel int8 - Topics []ListOffsetsRequestTopicV1 + ReplicaId int32 `json:"replicaId"` + IsolationLevel int8 `json:"isolationLevel"` + Topics []ListOffsetsRequestTopicV1 `json:"topics"` } // ListOffsets Request (Version: 4) type ListOffsetsRequestPartitionV4 struct { - PartitionIndex int32 - CurrentLeaderEpoch int32 - Timestamp int64 + PartitionIndex int32 `json:"partitionIndex"` + CurrentLeaderEpoch int32 `json:"currentLeaderEpoch"` + Timestamp int64 `json:"timestamp"` } type ListOffsetsRequestTopicV4 struct { - Name string - Partitions []ListOffsetsRequestPartitionV4 + Name string `json:"name"` + Partitions []ListOffsetsRequestPartitionV4 `json:"partitions"` } type ListOffsetsRequestV4 struct { - ReplicaId int32 - Topics []ListOffsetsRequestTopicV4 + ReplicaId int32 `json:"replicaId"` + Topics []ListOffsetsRequestTopicV4 `json:"topics"` } // ListOffsets Response (Version: 0) type ListOffsetsResponsePartitionV0 struct { - PartitionIndex int32 - ErrorCode int16 - OldStyleOffsets int64 + PartitionIndex int32 `json:"partitionIndex"` + ErrorCode int16 `json:"errorCode"` + OldStyleOffsets int64 `json:"oldStyleOffsets"` } type ListOffsetsResponseTopicV0 struct { - Name string - Partitions []ListOffsetsResponsePartitionV0 + Name string `json:"name"` + Partitions []ListOffsetsResponsePartitionV0 `json:"partitions"` } type ListOffsetsResponseV0 struct { - Topics []ListOffsetsResponseTopicV0 + Topics []ListOffsetsResponseTopicV0 `json:"topics"` } // ListOffsets Response (Version: 1) type ListOffsetsResponsePartitionV1 struct { - PartitionIndex int32 - ErrorCode int16 - Timestamp int64 - Offset int64 + PartitionIndex int32 `json:"partitionIndex"` + ErrorCode int16 `json:"errorCode"` + Timestamp int64 `json:"timestamp"` + Offset int64 `json:"offset"` } type ListOffsetsResponseTopicV1 struct { - Name string - Partitions []ListOffsetsResponsePartitionV1 + Name string `json:"name"` + Partitions []ListOffsetsResponsePartitionV1 `json:"partitions"` } type ListOffsetsResponseV1 struct { - Topics []ListOffsetsResponseTopicV1 + Topics []ListOffsetsResponseTopicV1 `json:"topics"` } // ListOffsets Response (Version: 2) type ListOffsetsResponseV2 struct { - ThrottleTimeMs int32 - Topics []ListOffsetsResponseTopicV1 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Topics []ListOffsetsResponseTopicV1 `json:"topics"` } // ListOffsets Response (Version: 4) type ListOffsetsResponsePartitionV4 struct { - PartitionIndex int32 - ErrorCode int16 - Timestamp int64 - Offset int64 - LeaderEpoch int32 + PartitionIndex int32 `json:"partitionIndex"` + ErrorCode int16 `json:"errorCode"` + Timestamp int64 `json:"timestamp"` + Offset int64 `json:"offset"` + LeaderEpoch int32 `json:"leaderEpoch"` } type ListOffsetsResponseTopicV4 struct { - Name string - Partitions []ListOffsetsResponsePartitionV4 + Name string `json:"name"` + Partitions []ListOffsetsResponsePartitionV4 `json:"partitions"` } type ListOffsetsResponseV4 struct { - Topics []ListOffsetsResponseTopicV4 + Topics []ListOffsetsResponseTopicV4 `json:"topics"` } // CreateTopics Request (Version: 0) type AssignmentsV0 struct { - PartitionIndex int32 - BrokerIds []int32 + PartitionIndex int32 `json:"partitionIndex"` + BrokerIds []int32 `json:"brokerIds"` } type CreateTopicsRequestConfigsV0 struct { - Name string - Value string + Name string `json:"name"` + Value string `json:"value"` } type CreateTopicsRequestTopicV0 struct { - Name string - NumPartitions int32 - ReplicationFactor int16 - Assignments []AssignmentsV0 - Configs []CreateTopicsRequestConfigsV0 + Name string `json:"name"` + NumPartitions int32 `json:"numPartitions"` + ReplicationFactor int16 `json:"replicationFactor"` + Assignments []AssignmentsV0 `json:"assignments"` + Configs []CreateTopicsRequestConfigsV0 `json:"configs"` } type CreateTopicsRequestV0 struct { - Topics []CreateTopicsRequestTopicV0 - TimeoutMs int32 + Topics []CreateTopicsRequestTopicV0 `json:"topics"` + TimeoutMs int32 `json:"timeoutMs"` } // CreateTopics Request (Version: 1) type CreateTopicsRequestV1 struct { - Topics []CreateTopicsRequestTopicV0 - TimeoutMs int32 - ValidateOnly bool + Topics []CreateTopicsRequestTopicV0 `json:"topics"` + TimeoutMs int32 `json:"timeoutMs"` + ValidateOnly bool `json:"validateOnly"` } // CreateTopics Response (Version: 0) type CreateTopicsResponseTopicV0 struct { - Name string - ErrorCode int16 + Name string `json:"name"` + ErrorCode int16 `json:"errorCode"` } type CreateTopicsResponseV0 struct { - Topics []CreateTopicsResponseTopicV0 + Topics []CreateTopicsResponseTopicV0 `json:"topics"` } // CreateTopics Response (Version: 1) type CreateTopicsResponseTopicV1 struct { - Name string - ErrorCode int16 - ErrorMessage string + Name string `json:"name"` + ErrorCode int16 `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` } type CreateTopicsResponseV1 struct { - Topics []CreateTopicsResponseTopicV1 + Topics []CreateTopicsResponseTopicV1 `json:"topics"` } // CreateTopics Response (Version: 2) type CreateTopicsResponseV2 struct { - ThrottleTimeMs int32 - Topics []CreateTopicsResponseTopicV1 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Topics []CreateTopicsResponseTopicV1 `json:"topics"` } // CreateTopics Response (Version: 5) type CreateTopicsResponseConfigsV5 struct { - Name string - Value string - ReadOnly bool - ConfigSource int8 - IsSensitive bool + Name string `json:"name"` + Value string `json:"value"` + ReadOnly bool `json:"readOnly"` + ConfigSource int8 `json:"configSource"` + IsSensitive bool `json:"isSensitive"` } type CreateTopicsResponseTopicV5 struct { - Name string - ErrorCode int16 - ErrorMessage string - NumPartitions int32 - ReplicationFactor int16 - Configs []CreateTopicsResponseConfigsV5 + Name string `json:"name"` + ErrorCode int16 `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` + NumPartitions int32 `json:"numPartitions"` + ReplicationFactor int16 `json:"replicationFactor"` + Configs []CreateTopicsResponseConfigsV5 `json:"configs"` } type CreateTopicsResponseV5 struct { - ThrottleTimeMs int32 - Topics []CreateTopicsResponseTopicV5 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Topics []CreateTopicsResponseTopicV5 `json:"topics"` } // CreateTopics Response (Version: 7) type CreateTopicsResponseTopicV7 struct { - Name string - TopicID UUID - ErrorCode int16 - ErrorMessage string - NumPartitions int32 - ReplicationFactor int16 - Configs []CreateTopicsResponseConfigsV5 + Name string `json:"name"` + TopicID UUID `json:"topicID"` + ErrorCode int16 `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` + NumPartitions int32 `json:"numPartitions"` + ReplicationFactor int16 `json:"replicationFactor"` + Configs []CreateTopicsResponseConfigsV5 `json:"configs"` } type CreateTopicsResponseV7 struct { - ThrottleTimeMs int32 - Topics []CreateTopicsResponseTopicV7 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Topics []CreateTopicsResponseTopicV7 `json:"topics"` } // DeleteTopics Request (Version: 0) type DeleteTopicsRequestV0 struct { - TopicNames []string - TimeoutMs int32 + TopicNames []string `json:"topicNames"` + TimeoutMs int32 `json:"timeoutMs"` } // DeleteTopics Request (Version: 6) type DeleteTopicsRequestTopicV6 struct { - Name string - UUID UUID + Name string `json:"name"` + UUID UUID `json:"uuid"` } type DeleteTopicsRequestV6 struct { - Topics []DeleteTopicsRequestTopicV6 - TimeoutMs int32 + Topics []DeleteTopicsRequestTopicV6 `json:"topics"` + TimeoutMs int32 `json:"timeoutMs"` } // DeleteTopics Response (Version: 0) type DeleteTopicsReponseResponseV0 struct { - Name string - ErrorCode int16 + Name string `json:"name"` + ErrorCode int16 `json:"errorCode"` } type DeleteTopicsReponseV0 struct { - Responses []DeleteTopicsReponseResponseV0 + Responses []DeleteTopicsReponseResponseV0 `json:"responses"` } // DeleteTopics Response (Version: 1) type DeleteTopicsReponseV1 struct { - ThrottleTimeMs int32 - Responses []DeleteTopicsReponseResponseV0 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Responses []DeleteTopicsReponseResponseV0 `json:"responses"` } // DeleteTopics Response (Version: 5) type DeleteTopicsReponseResponseV5 struct { - Name string - ErrorCode int16 - ErrorMessage string + Name string `json:"name"` + ErrorCode int16 `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` } type DeleteTopicsReponseV5 struct { - ThrottleTimeMs int32 - Responses []DeleteTopicsReponseResponseV5 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Responses []DeleteTopicsReponseResponseV5 `json:"responses"` } // DeleteTopics Response (Version: 6) type DeleteTopicsReponseResponseV6 struct { - Name string - TopicID UUID - ErrorCode int16 - ErrorMessage string + Name string `json:"name"` + TopicID UUID `json:"topicID"` + ErrorCode int16 `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` } type DeleteTopicsReponseV6 struct { - ThrottleTimeMs int32 - Responses []DeleteTopicsReponseResponseV6 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Responses []DeleteTopicsReponseResponseV6 `json:"responses"` } diff --git a/tap/extensions/redis/helpers.go b/tap/extensions/redis/helpers.go index a020b722d..3f8b61791 100644 --- a/tap/extensions/redis/helpers.go +++ b/tap/extensions/redis/helpers.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "github.com/up9inc/mizu/tap/api" ) @@ -24,33 +25,38 @@ type RedisWrapper struct { Details interface{} `json:"details"` } -func representGeneric(generic map[string]interface{}) (representation []interface{}) { - details, _ := json.Marshal([]map[string]string{ +func representGeneric(generic map[string]interface{}, selectorPrefix string) (representation []interface{}) { + details, _ := json.Marshal([]api.TableData{ { - "name": "Type", - "value": generic["type"].(string), + Name: "Type", + Value: generic["type"].(string), + Selector: fmt.Sprintf("%stype", selectorPrefix), }, { - "name": "Command", - "value": generic["command"].(string), + Name: "Command", + Value: generic["command"].(string), + Selector: fmt.Sprintf("%scommand", selectorPrefix), }, { - "name": "Key", - "value": generic["key"].(string), + Name: "Key", + Value: generic["key"].(string), + Selector: fmt.Sprintf("%skey", selectorPrefix), }, { - "name": "Value", - "value": generic["value"].(string), + Name: "Value", + Value: generic["value"].(string), + Selector: fmt.Sprintf("%svalue", selectorPrefix), }, { - "name": "Keyword", - "value": generic["keyword"].(string), + Name: "Keyword", + Value: generic["keyword"].(string), + Selector: fmt.Sprintf("%skeyword", selectorPrefix), }, }) - representation = append(representation, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + representation = append(representation, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) return diff --git a/tap/extensions/redis/main.go b/tap/extensions/redis/main.go index be0650bfc..e8a6d0f31 100644 --- a/tap/extensions/redis/main.go +++ b/tap/extensions/redis/main.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "log" + "time" "github.com/up9inc/mizu/tap/api" ) @@ -13,6 +14,7 @@ var protocol api.Protocol = api.Protocol{ Name: "redis", LongName: "Redis Serialization Protocol", Abbreviation: "REDIS", + Macro: "redis", Version: "3.x", BackgroundColor: "#a41e11", ForegroundColor: "#ffffff", @@ -34,7 +36,7 @@ func (d dissecting) Register(extension *api.Extension) { } func (d dissecting) Ping() { - log.Printf("pong %s\n", protocol.Name) + log.Printf("pong %s", protocol.Name) } func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error { @@ -57,15 +59,11 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co } } -func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry { +func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry { request := item.Pair.Request.Payload.(map[string]interface{}) + response := item.Pair.Response.Payload.(map[string]interface{}) reqDetails := request["details"].(map[string]interface{}) - service := "redis" - if resolvedDestination != "" { - service = resolvedDestination - } else if resolvedSource != "" { - service = resolvedSource - } + resDetails := response["details"].(map[string]interface{}) method := "" if reqDetails["command"] != nil { @@ -78,54 +76,48 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve } request["url"] = summary - entryBytes, _ := json.Marshal(item.Pair) + elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds() + if elapsedTime < 0 { + elapsedTime = 0 + } return &api.MizuEntry{ - ProtocolName: protocol.Name, - ProtocolLongName: protocol.LongName, - ProtocolAbbreviation: protocol.Abbreviation, - ProtocolVersion: protocol.Version, - ProtocolBackgroundColor: protocol.BackgroundColor, - ProtocolForegroundColor: protocol.ForegroundColor, - ProtocolFontSize: protocol.FontSize, - ProtocolReferenceLink: protocol.ReferenceLink, - EntryId: entryId, - Entry: string(entryBytes), - Url: fmt.Sprintf("%s%s", service, summary), - Method: method, - Status: 0, - RequestSenderIp: item.ConnectionInfo.ClientIP, - Service: service, - Timestamp: item.Timestamp, - ElapsedTime: 0, - Path: summary, - ResolvedSource: resolvedSource, - ResolvedDestination: resolvedDestination, - SourceIp: item.ConnectionInfo.ClientIP, - DestinationIp: item.ConnectionInfo.ServerIP, - SourcePort: item.ConnectionInfo.ClientPort, - DestinationPort: item.ConnectionInfo.ServerPort, - IsOutgoing: item.ConnectionInfo.IsOutgoing, + Protocol: protocol, + Source: &api.TCP{ + Name: resolvedSource, + IP: item.ConnectionInfo.ClientIP, + Port: item.ConnectionInfo.ClientPort, + }, + Destination: &api.TCP{ + Name: resolvedDestination, + IP: item.ConnectionInfo.ServerIP, + Port: item.ConnectionInfo.ServerPort, + }, + Outgoing: item.ConnectionInfo.IsOutgoing, + Request: reqDetails, + Response: resDetails, + Method: method, + Status: 0, + Timestamp: item.Timestamp, + StartTime: item.Pair.Request.CaptureTime, + ElapsedTime: elapsedTime, + Summary: summary, + IsOutgoing: item.ConnectionInfo.IsOutgoing, } } func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { return &api.BaseEntryDetails{ - Id: entry.EntryId, - Protocol: protocol, - Url: entry.Url, - RequestSenderIp: entry.RequestSenderIp, - Service: entry.Service, - Summary: entry.Path, - StatusCode: entry.Status, - Method: entry.Method, - Timestamp: entry.Timestamp, - SourceIp: entry.SourceIp, - DestinationIp: entry.DestinationIp, - SourcePort: entry.SourcePort, - DestinationPort: entry.DestinationPort, - IsOutgoing: entry.IsOutgoing, - Latency: entry.ElapsedTime, + Id: entry.Id, + Protocol: protocol, + Summary: entry.Summary, + StatusCode: entry.Status, + Method: entry.Method, + Timestamp: entry.Timestamp, + Source: entry.Source, + Destination: entry.Destination, + IsOutgoing: entry.IsOutgoing, + Latency: entry.ElapsedTime, Rules: api.ApplicableRules{ Latency: 0, Status: false, @@ -133,22 +125,21 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { } } -func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) { - p = protocol +func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) { bodySize = 0 - var root map[string]interface{} - json.Unmarshal([]byte(entry.Entry), &root) representation := make(map[string]interface{}, 0) - request := root["request"].(map[string]interface{})["payload"].(map[string]interface{}) - response := root["response"].(map[string]interface{})["payload"].(map[string]interface{}) - reqDetails := request["details"].(map[string]interface{}) - resDetails := response["details"].(map[string]interface{}) - repRequest := representGeneric(reqDetails) - repResponse := representGeneric(resDetails) + repRequest := representGeneric(request, `request.`) + repResponse := representGeneric(response, `response.`) representation["request"] = repRequest representation["response"] = repResponse object, err = json.Marshal(representation) return } +func (d dissecting) Macros() map[string]string { + return map[string]string{ + `redis`: fmt.Sprintf(`proto.name == "%s"`, protocol.Name), + } +} + var Dissector dissecting diff --git a/tap/extensions/redis/read.go b/tap/extensions/redis/read.go index 2732970a1..f93fa5a7e 100644 --- a/tap/extensions/redis/read.go +++ b/tap/extensions/redis/read.go @@ -313,7 +313,7 @@ func (p *RedisProtocol) Read() (packet *RedisPacket, err error) { packet.Value = fmt.Sprintf("%s]", packet.Value) } default: - msg := fmt.Sprintf("Unrecognized element in Redis array: %v\n", reflect.TypeOf(array[0])) + msg := fmt.Sprintf("Unrecognized element in Redis array: %v", reflect.TypeOf(array[0])) err = errors.New(msg) return } @@ -333,7 +333,7 @@ func (p *RedisProtocol) Read() (packet *RedisPacket, err error) { case int64: packet.Value = fmt.Sprintf("%d", x.(int64)) default: - msg := fmt.Sprintf("Unrecognized Redis data type: %v\n", reflect.TypeOf(x)) + msg := fmt.Sprintf("Unrecognized Redis data type: %v", reflect.TypeOf(x)) err = errors.New(msg) return } diff --git a/tap/go.mod b/tap/go.mod index 12b93ef10..a6ce9aad0 100644 --- a/tap/go.mod +++ b/tap/go.mod @@ -4,13 +4,11 @@ go 1.16 require ( github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 - github.com/go-errors/errors v1.4.1 github.com/google/gopacket v1.1.19 - github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 github.com/up9inc/mizu/shared v0.0.0 github.com/up9inc/mizu/tap/api v0.0.0 - golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 // indirect - golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 // indirect + github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f + k8s.io/api v0.21.2 ) replace github.com/up9inc/mizu/tap/api v0.0.0 => ./api diff --git a/tap/go.sum b/tap/go.sum index 965486e84..a33956565 100644 --- a/tap/go.sum +++ b/tap/go.sum @@ -1,34 +1,708 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M= github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/go-errors/errors v1.4.1 h1:IvVlgbzSsaUNudsw5dcXSzF3EWyXTi5XrAdngnuhRyg= -github.com/go-errors/errors v1.4.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= +github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= +github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= +github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= +github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= +github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= +github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y= +k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU= +k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc= +k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM= +k8s.io/cli-runtime v0.21.2/go.mod h1:8u/jFcM0QpoI28f6sfrAAIslLCXUYKD5SsPPMWiHYrI= +k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA= +k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U= +k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc= +k8s.io/component-helpers v0.21.2/go.mod h1:DbyFt/A0p6Cv+R5+QOGSJ5f5t4xDfI8Yb89a57DgJlQ= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= +k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= +k8s.io/kubectl v0.21.2/go.mod h1:PgeUclpG8VVmmQIl8zpLar3IQEpFc9mrmvlwY3CK1xo= +k8s.io/metrics v0.21.2/go.mod h1:wzlOINZMCtWq8dR9gHlyaOemmYlOpAoldEIXE82gAhI= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/kustomize/api v0.8.8/go.mod h1:He1zoK0nk43Pc6NlV085xDXDXTNprtcyKZVm3swsdNY= +sigs.k8s.io/kustomize/cmd/config v0.9.10/go.mod h1:Mrby0WnRH7hA6OwOYnYpfpiY0WJIMgYrEDfwOeFdMK0= +sigs.k8s.io/kustomize/kustomize/v4 v4.1.2/go.mod h1:PxBvo4WGYlCLeRPL+ziT64wBXqbgfcalOS/SXa/tcyo= +sigs.k8s.io/kustomize/kyaml v0.10.17/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4cPcfUVt5Hg= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= +sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/tap/passive_tapper.go b/tap/passive_tapper.go index 6dfe1390a..275f0288d 100644 --- a/tap/passive_tapper.go +++ b/tap/passive_tapper.go @@ -20,6 +20,7 @@ import ( "github.com/up9inc/mizu/tap/api" "github.com/up9inc/mizu/tap/diagnose" "github.com/up9inc/mizu/tap/source" + v1 "k8s.io/api/core/v1" ) const cleanPeriod = time.Second * 10 @@ -40,6 +41,7 @@ var verbose = flag.Bool("verbose", false, "Be verbose") var debug = flag.Bool("debug", false, "Display debug information") var quiet = flag.Bool("quiet", false, "Be quiet regarding errors") var hexdumppkt = flag.Bool("dumppkt", false, "Dump packet as hex") +var procfs = flag.String("procfs", "/proc", "The procfs directory, used when mapping host volumes into a container") // capture var iface = flag.String("i", "en0", "Interface to read packets from") @@ -48,14 +50,16 @@ var snaplen = flag.Int("s", 65536, "Snap length (number of bytes max to read per var tstype = flag.String("timestamp_type", "", "Type of timestamps to use") var promisc = flag.Bool("promisc", true, "Set promiscuous mode") var staleTimeoutSeconds = flag.Int("staletimout", 120, "Max time in seconds to keep connections which don't transmit data") +var pids = flag.String("pids", "", "A comma separated list of PIDs to capture their network namespaces") +var istio = flag.Bool("istio", false, "Record decrypted traffic if the cluster configured with istio and mtls") var memprofile = flag.String("memprofile", "", "Write memory profile") type TapOpts struct { - HostMode bool + HostMode bool + FilterAuthorities []v1.Pod } -var hostMode bool // global var extensions []*api.Extension // global var filteringOptions *api.TrafficFilteringOptions // global @@ -78,15 +82,18 @@ func inArrayString(arr []string, valueToCheck string) bool { } func StartPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem, extensionsRef []*api.Extension, options *api.TrafficFilteringOptions) { - hostMode = opts.HostMode extensions = extensionsRef filteringOptions = options + if opts.FilterAuthorities == nil { + opts.FilterAuthorities = []v1.Pod{} + } + if GetMemoryProfilingEnabled() { diagnose.StartMemoryProfiler(os.Getenv(MemoryProfilingDumpPath), os.Getenv(MemoryProfilingTimeIntervalSeconds)) } - go startPassiveTapper(outputItems) + go startPassiveTapper(opts, outputItems) } func printPeriodicStats(cleaner *Cleaner) { @@ -129,44 +136,49 @@ func printPeriodicStats(cleaner *Cleaner) { } } -func startPassiveTapper(outputItems chan *api.OutputChannelItem) { - streamsMap := NewTcpStreamMap() - go streamsMap.closeTimedoutTcpStreamChannels() - - diagnose.InitializeErrorsMap(*debug, *verbose, *quiet) - diagnose.InitializeTapperInternalStats() - +func initializePacketSources(opts *TapOpts) (*source.PacketSourceManager, error) { var bpffilter string if len(flag.Args()) > 0 { bpffilter = strings.Join(flag.Args(), " ") } - packetSource, err := source.NewTcpPacketSource(*fname, *iface, source.TcpPacketSourceBehaviour{ + behaviour := source.TcpPacketSourceBehaviour{ SnapLength: *snaplen, Promisc: *promisc, Tstype: *tstype, DecoderName: *decoder, Lazy: *lazy, BpfFilter: bpffilter, - }) + } + + return source.NewPacketSourceManager(*procfs, *pids, *fname, *iface, *istio, opts.FilterAuthorities, behaviour) +} + +func startPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem) { + streamsMap := NewTcpStreamMap() + go streamsMap.closeTimedoutTcpStreamChannels() + + diagnose.InitializeErrorsMap(*debug, *verbose, *quiet) + diagnose.InitializeTapperInternalStats() + + sources, err := initializePacketSources(opts) if err != nil { logger.Log.Fatal(err) } - defer packetSource.Close() + defer sources.Close() if err != nil { logger.Log.Fatal(err) } packets := make(chan source.TcpPacketInfo) - assembler := NewTcpAssembler(outputItems, streamsMap) + assembler := NewTcpAssembler(outputItems, streamsMap, opts) - logger.Log.Info("Starting to read packets") diagnose.AppStats.SetStartTime(time.Now()) - go packetSource.ReadPackets(!*nodefrag, packets) + sources.ReadPackets(!*nodefrag, packets) staleConnectionTimeout := time.Second * time.Duration(*staleTimeoutSeconds) cleaner := Cleaner{ @@ -186,7 +198,7 @@ func startPassiveTapper(outputItems chan *api.OutputChannelItem) { } if err := diagnose.DumpMemoryProfile(*memprofile); err != nil { - logger.Log.Errorf("Error dumping memory profile %v\n", err) + logger.Log.Errorf("Error dumping memory profile %v", err) } assembler.waitAndDump() diff --git a/tap/settings.go b/tap/settings.go index 4e3b6c4a4..b9e25da6d 100644 --- a/tap/settings.go +++ b/tap/settings.go @@ -18,24 +18,6 @@ const ( TcpStreamChannelTimeoutMsDefaultValue = 10000 ) -type globalSettings struct { - filterAuthorities []string -} - -var gSettings = &globalSettings{ - filterAuthorities: []string{}, -} - -func SetFilterAuthorities(ipAddresses []string) { - gSettings.filterAuthorities = ipAddresses -} - -func GetFilterIPs() []string { - addresses := make([]string, len(gSettings.filterAuthorities)) - copy(addresses, gSettings.filterAuthorities) - return addresses -} - func GetMaxBufferedPagesTotal() int { valueFromEnv, err := strconv.Atoi(os.Getenv(MaxBufferedPagesTotalEnvVarName)) if err != nil { diff --git a/tap/source/envoy_discoverer.go b/tap/source/envoy_discoverer.go new file mode 100644 index 000000000..0230b1273 --- /dev/null +++ b/tap/source/envoy_discoverer.go @@ -0,0 +1,113 @@ +package source + +import ( + "fmt" + "io/ioutil" + "os" + "regexp" + "strings" + + "github.com/up9inc/mizu/shared/logger" + v1 "k8s.io/api/core/v1" +) + +const envoyBinary = "/envoy" + +var numberRegex = regexp.MustCompile("[0-9]+") + +func discoverRelevantEnvoyPids(procfs string, pods []v1.Pod) ([]string, error) { + result := make([]string, 0) + + pids, err := ioutil.ReadDir(procfs) + + if err != nil { + return result, err + } + + logger.Log.Infof("Starting envoy auto discoverer %v %v - scanning %v potential pids", + procfs, pods, len(pids)) + + for _, pid := range pids { + if !pid.IsDir() { + continue + } + + if !numberRegex.MatchString(pid.Name()) { + continue + } + + if checkPid(procfs, pid.Name(), pods) { + result = append(result, pid.Name()) + } + } + + logger.Log.Infof("Found %v relevant envoy processes - %v", len(result), result) + + return result, nil +} + +func checkPid(procfs string, pid string, pods []v1.Pod) bool { + execLink := fmt.Sprintf("%v/%v/exe", procfs, pid) + exec, err := os.Readlink(execLink) + + if err != nil { + // Debug on purpose - it may happen due to many reasons and we only care + // for it during troubleshooting + // + logger.Log.Debugf("Unable to read link %v - %v\n", execLink, err) + return false + } + + if !strings.HasSuffix(exec, envoyBinary) { + return false + } + + environmentFile := fmt.Sprintf("%v/%v/environ", procfs, pid) + podIp, err := readEnvironmentVariable(environmentFile, "INSTANCE_IP") + + if err != nil { + return false + } + + if podIp == "" { + logger.Log.Debugf("Found an envoy process without INSTANCE_IP variable %v\n", pid) + return false + } + + logger.Log.Infof("Found envoy pid %v with cluster ip %v", pid, podIp) + + for _, pod := range pods { + if pod.Status.PodIP == podIp { + return true + } + } + + return false +} + +func readEnvironmentVariable(file string, name string) (string, error) { + bytes, err := ioutil.ReadFile(file) + + if err != nil { + logger.Log.Warningf("Error reading environment file %v - %v", file, err) + return "", err + } + + envs := strings.Split(string(bytes), string([]byte{0})) + + for _, env := range envs { + if !strings.Contains(env, "=") { + continue + } + + parts := strings.Split(env, "=") + varName := parts[0] + value := parts[1] + + if name == varName { + return value, nil + } + } + + return "", nil +} diff --git a/tap/source/packet_source_manager.go b/tap/source/packet_source_manager.go new file mode 100644 index 000000000..af88554ea --- /dev/null +++ b/tap/source/packet_source_manager.go @@ -0,0 +1,192 @@ +package source + +import ( + "fmt" + "runtime" + "strconv" + "strings" + + "github.com/up9inc/mizu/shared/logger" + "github.com/vishvananda/netns" + v1 "k8s.io/api/core/v1" +) + +type PacketSourceManager struct { + sources []*tcpPacketSource +} + +func NewPacketSourceManager(procfs string, pids string, filename string, interfaceName string, + istio bool, pods []v1.Pod, behaviour TcpPacketSourceBehaviour) (*PacketSourceManager, error) { + sources := make([]*tcpPacketSource, 0) + sources, err := createHostSource(sources, filename, interfaceName, behaviour) + + if err != nil { + return nil, err + } + + sources = createSourcesFromPids(sources, procfs, pids, interfaceName, behaviour) + sources = createSourcesFromEnvoy(sources, istio, procfs, pods, interfaceName, behaviour) + + return &PacketSourceManager{ + sources: sources, + }, nil +} + +func createHostSource(sources []*tcpPacketSource, filename string, interfaceName string, + behaviour TcpPacketSourceBehaviour) ([]*tcpPacketSource, error) { + hostSource, err := newHostPacketSource(filename, interfaceName, behaviour) + + if err != nil { + return sources, err + } + + return append(sources, hostSource), nil +} + +func createSourcesFromPids(sources []*tcpPacketSource, procfs string, pids string, + interfaceName string, behaviour TcpPacketSourceBehaviour) []*tcpPacketSource { + if pids == "" { + return sources + } + + netnsSources := newNetnsPacketSources(procfs, strings.Split(pids, ","), interfaceName, behaviour) + sources = append(sources, netnsSources...) + return sources +} + +func createSourcesFromEnvoy(sources []*tcpPacketSource, istio bool, procfs string, clusterIps []v1.Pod, + interfaceName string, behaviour TcpPacketSourceBehaviour) []*tcpPacketSource { + if !istio { + return sources + } + + envoyPids, err := discoverRelevantEnvoyPids(procfs, clusterIps) + + if err != nil { + logger.Log.Warningf("Unable to discover envoy pids - %v", err) + return sources + } + + netnsSources := newNetnsPacketSources(procfs, envoyPids, interfaceName, behaviour) + sources = append(sources, netnsSources...) + + return sources +} + +func newHostPacketSource(filename string, interfaceName string, + behaviour TcpPacketSourceBehaviour) (*tcpPacketSource, error) { + var name string + + if filename == "" { + name = fmt.Sprintf("host-%v", interfaceName) + } else { + name = fmt.Sprintf("file-%v", filename) + } + + source, err := newTcpPacketSource(name, filename, interfaceName, behaviour) + + if err != nil { + return nil, err + } + + return source, nil +} + +func newNetnsPacketSources(procfs string, pids []string, interfaceName string, + behaviour TcpPacketSourceBehaviour) []*tcpPacketSource { + result := make([]*tcpPacketSource, 0) + + for _, pidstr := range pids { + pid, err := strconv.Atoi(pidstr) + + if err != nil { + logger.Log.Errorf("Invalid PID: %v - %v", pid, err) + continue + } + + nsh, err := netns.GetFromPath(fmt.Sprintf("%v/%v/ns/net", procfs, pid)) + + if err != nil { + logger.Log.Errorf("Unable to get netns of pid %v - %v", pid, err) + continue + } + + src, err := newNetnsPacketSource(pid, nsh, interfaceName, behaviour) + + if err != nil { + logger.Log.Errorf("Error starting netns packet source for %v - %v", pid, err) + continue + } + + result = append(result, src) + } + + return result +} + +func newNetnsPacketSource(pid int, nsh netns.NsHandle, interfaceName string, + behaviour TcpPacketSourceBehaviour) (*tcpPacketSource, error) { + + done := make(chan *tcpPacketSource) + errors := make(chan error) + + go func(done chan<- *tcpPacketSource) { + // Setting a netns should be done from a dedicated OS thread. + // + // goroutines are not really OS threads, we try to mimic the issue by + // locking the OS thread to this goroutine + // + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + oldnetns, err := netns.Get() + + if err != nil { + logger.Log.Errorf("Unable to get netns of current thread %v", err) + errors <- err + return + } + + if err := netns.Set(nsh); err != nil { + logger.Log.Errorf("Unable to set netns of pid %v - %v", pid, err) + errors <- err + return + } + + name := fmt.Sprintf("netns-%v-%v", pid, interfaceName) + src, err := newTcpPacketSource(name, "", interfaceName, behaviour) + + if err != nil { + logger.Log.Errorf("Error listening to PID %v - %v", pid, err) + errors <- err + return + } + + if err := netns.Set(oldnetns); err != nil { + logger.Log.Errorf("Unable to set back netns of current thread %v", err) + errors <- err + return + } + + done <- src + }(done) + + select { + case err := <-errors: + return nil, err + case source := <-done: + return source, nil + } +} + +func (m *PacketSourceManager) ReadPackets(ipdefrag bool, packets chan<- TcpPacketInfo) { + for _, src := range m.sources { + go src.readPackets(ipdefrag, packets) + } +} + +func (m *PacketSourceManager) Close() { + for _, src := range m.sources { + src.close() + } +} diff --git a/tap/source/tcp_packet_source.go b/tap/source/tcp_packet_source.go index 61ff7446d..a7c0258ad 100644 --- a/tap/source/tcp_packet_source.go +++ b/tap/source/tcp_packet_source.go @@ -13,11 +13,12 @@ import ( "github.com/up9inc/mizu/tap/diagnose" ) -type TcpPacketSource struct { +type tcpPacketSource struct { source *gopacket.PacketSource handle *pcap.Handle defragger *ip4defrag.IPv4Defragmenter Behaviour *TcpPacketSourceBehaviour + name string } type TcpPacketSourceBehaviour struct { @@ -31,14 +32,15 @@ type TcpPacketSourceBehaviour struct { type TcpPacketInfo struct { Packet gopacket.Packet - Source *TcpPacketSource + Source *tcpPacketSource } -func NewTcpPacketSource(filename string, interfaceName string, - behaviour TcpPacketSourceBehaviour) (*TcpPacketSource, error) { +func newTcpPacketSource(name, filename string, interfaceName string, + behaviour TcpPacketSourceBehaviour) (*tcpPacketSource, error) { var err error - result := &TcpPacketSource{ + result := &tcpPacketSource{ + name: name, defragger: ip4defrag.NewIPv4Defragmenter(), Behaviour: &behaviour, } @@ -96,49 +98,50 @@ func NewTcpPacketSource(filename string, interfaceName string, return result, nil } -func (source *TcpPacketSource) Close() { +func (source *tcpPacketSource) close() { if source.handle != nil { source.handle.Close() } } -func (source *TcpPacketSource) ReadPackets(ipdefrag bool, packets chan<- TcpPacketInfo) error { +func (source *tcpPacketSource) readPackets(ipdefrag bool, packets chan<- TcpPacketInfo) { + logger.Log.Infof("Start reading packets from %v", source.name) + for { packet, err := source.source.NextPacket() if err == io.EOF { - return err + logger.Log.Infof("Got EOF while reading packets from %v", source.name) + return } else if err != nil { if err.Error() != "Timeout Expired" { - logger.Log.Debugf("Error: %T", err) + logger.Log.Debugf("Error while reading from %v - %v", source.name, err) } continue } // defrag the IPv4 packet if required - if !ipdefrag { - ip4Layer := packet.Layer(layers.LayerTypeIPv4) - if ip4Layer == nil { - continue - } - ip4 := ip4Layer.(*layers.IPv4) - l := ip4.Length - newip4, err := source.defragger.DefragIPv4(ip4) - if err != nil { - logger.Log.Fatal("Error while de-fragmenting", err) - } else if newip4 == nil { - logger.Log.Debugf("Fragment...") - continue // packet fragment, we don't have whole packet yet. - } - if newip4.Length != l { - diagnose.InternalStats.Ipdefrag++ - logger.Log.Debugf("Decoding re-assembled packet: %s", newip4.NextLayerType()) - pb, ok := packet.(gopacket.PacketBuilder) - if !ok { - logger.Log.Panic("Not a PacketBuilder") + if ipdefrag { + if ip4Layer := packet.Layer(layers.LayerTypeIPv4); ip4Layer != nil { + ip4 := ip4Layer.(*layers.IPv4) + l := ip4.Length + newip4, err := source.defragger.DefragIPv4(ip4) + if err != nil { + logger.Log.Fatal("Error while de-fragmenting", err) + } else if newip4 == nil { + logger.Log.Debugf("Fragment...") + continue // packet fragment, we don't have whole packet yet. + } + if newip4.Length != l { + diagnose.InternalStats.Ipdefrag++ + logger.Log.Debugf("Decoding re-assembled packet: %s", newip4.NextLayerType()) + pb, ok := packet.(gopacket.PacketBuilder) + if !ok { + logger.Log.Panic("Not a PacketBuilder") + } + nextDecoder := newip4.NextLayerType() + _ = nextDecoder.Decode(newip4.Payload, pb) } - nextDecoder := newip4.NextLayerType() - _ = nextDecoder.Decode(newip4.Payload, pb) } } diff --git a/tap/tcp_assembler.go b/tap/tcp_assembler.go index 4dd05b095..150834967 100644 --- a/tap/tcp_assembler.go +++ b/tap/tcp_assembler.go @@ -16,6 +16,8 @@ import ( "github.com/up9inc/mizu/tap/source" ) +const PACKETS_SEEN_LOG_THRESHOLD = 1000 + type tcpAssembler struct { *reassembly.Assembler streamPool *reassembly.StreamPool @@ -33,13 +35,13 @@ func (c *context) GetCaptureInfo() gopacket.CaptureInfo { return c.CaptureInfo } -func NewTcpAssembler(outputItems chan *api.OutputChannelItem, streamsMap *tcpStreamMap) *tcpAssembler { +func NewTcpAssembler(outputItems chan *api.OutputChannelItem, streamsMap *tcpStreamMap, opts *TapOpts) *tcpAssembler { var emitter api.Emitter = &api.Emitting{ AppStats: &diagnose.AppStats, OutputChannel: outputItems, } - streamFactory := NewTcpStreamFactory(emitter, streamsMap) + streamFactory := NewTcpStreamFactory(emitter, streamsMap, opts) streamPool := reassembly.NewStreamPool(streamFactory) assembler := reassembly.NewAssembler(streamPool) @@ -63,7 +65,11 @@ func (a *tcpAssembler) processPackets(dumpPacket bool, packets <-chan source.Tcp for packetInfo := range packets { packetsCount := diagnose.AppStats.IncPacketsCount() - logger.Log.Debugf("PACKET #%d", packetsCount) + + if packetsCount % PACKETS_SEEN_LOG_THRESHOLD == 0 { + logger.Log.Debugf("Packets seen: #%d", packetsCount) + } + packet := packetInfo.Packet data := packet.Data() diagnose.AppStats.UpdateProcessedBytes(uint64(len(data))) @@ -78,14 +84,14 @@ func (a *tcpAssembler) processPackets(dumpPacket bool, packets <-chan source.Tcp if *checksum { err := tcp.SetNetworkLayerForChecksum(packet.NetworkLayer()) if err != nil { - logger.Log.Fatalf("Failed to set network layer for checksum: %s\n", err) + logger.Log.Fatalf("Failed to set network layer for checksum: %s", err) } } c := context{ CaptureInfo: packet.Metadata().CaptureInfo, } diagnose.InternalStats.Totalsz += len(tcp.Payload) - logger.Log.Debugf("%s : %v -> %s : %v", packet.NetworkLayer().NetworkFlow().Src(), tcp.SrcPort, packet.NetworkLayer().NetworkFlow().Dst(), tcp.DstPort) + logger.Log.Debugf("%s:%v -> %s:%v", packet.NetworkLayer().NetworkFlow().Src(), tcp.SrcPort, packet.NetworkLayer().NetworkFlow().Dst(), tcp.DstPort) a.assemblerMutex.Lock() a.AssembleWithContext(packet.NetworkLayer().NetworkFlow(), tcp, &c) a.assemblerMutex.Unlock() diff --git a/tap/tcp_reader.go b/tap/tcp_reader.go index fff0a5a4e..8be3065b2 100644 --- a/tap/tcp_reader.go +++ b/tap/tcp_reader.go @@ -66,7 +66,7 @@ func (h *tcpReader) Read(p []byte) (int, error) { clientHello := tlsx.ClientHello{} err := clientHello.Unmarshall(msg.bytes) if err == nil { - logger.Log.Debugf("Detected TLS client hello with SNI %s\n", clientHello.SNI) + logger.Log.Debugf("Detected TLS client hello with SNI %s", clientHello.SNI) // TODO: Throws `panic: runtime error: invalid memory address or nil pointer dereference` error. // numericPort, _ := strconv.Atoi(h.tcpID.DstPort) // h.outboundLinkWriter.WriteOutboundLink(h.tcpID.SrcIP, h.tcpID.DstIP, numericPort, clientHello.SNI, TLSProtocol) diff --git a/tap/tcp_stream_factory.go b/tap/tcp_stream_factory.go index 5176f20aa..847ca10d1 100644 --- a/tap/tcp_stream_factory.go +++ b/tap/tcp_stream_factory.go @@ -7,6 +7,7 @@ import ( "github.com/up9inc/mizu/shared/logger" "github.com/up9inc/mizu/tap/api" + v1 "k8s.io/api/core/v1" "github.com/google/gopacket" "github.com/google/gopacket/layers" // pulls in all layers decoders @@ -24,6 +25,7 @@ type tcpStreamFactory struct { Emitter api.Emitter streamsMap *tcpStreamMap ownIps []string + opts *TapOpts } type tcpStreamWrapper struct { @@ -31,7 +33,7 @@ type tcpStreamWrapper struct { createdAt time.Time } -func NewTcpStreamFactory(emitter api.Emitter, streamsMap *tcpStreamMap) *tcpStreamFactory { +func NewTcpStreamFactory(emitter api.Emitter, streamsMap *tcpStreamMap, opts *TapOpts) *tcpStreamFactory { var ownIps []string if localhostIPs, err := getLocalhostIPs(); err != nil { @@ -47,6 +49,7 @@ func NewTcpStreamFactory(emitter api.Emitter, streamsMap *tcpStreamMap) *tcpStre Emitter: emitter, streamsMap: streamsMap, ownIps: ownIps, + opts: opts, } } @@ -138,18 +141,27 @@ func (factory *tcpStreamFactory) WaitGoRoutines() { factory.wg.Wait() } +func inArrayPod(pods []v1.Pod, address string) bool { + for _, pod := range pods { + if pod.Status.PodIP == address { + return true + } + } + return false +} + func (factory *tcpStreamFactory) getStreamProps(srcIP string, srcPort string, dstIP string, dstPort string) *streamProps { - if hostMode { - if inArrayString(gSettings.filterAuthorities, fmt.Sprintf("%s:%s", dstIP, dstPort)) { + if factory.opts.HostMode { + if inArrayPod(factory.opts.FilterAuthorities, fmt.Sprintf("%s:%s", dstIP, dstPort)) { logger.Log.Debugf("getStreamProps %s", fmt.Sprintf("+ host1 %s:%s", dstIP, dstPort)) return &streamProps{isTapTarget: true, isOutgoing: false} - } else if inArrayString(gSettings.filterAuthorities, dstIP) { + } else if inArrayPod(factory.opts.FilterAuthorities, dstIP) { logger.Log.Debugf("getStreamProps %s", fmt.Sprintf("+ host2 %s", dstIP)) return &streamProps{isTapTarget: true, isOutgoing: false} - } else if inArrayString(gSettings.filterAuthorities, fmt.Sprintf("%s:%s", srcIP, srcPort)) { + } else if inArrayPod(factory.opts.FilterAuthorities, fmt.Sprintf("%s:%s", srcIP, srcPort)) { logger.Log.Debugf("getStreamProps %s", fmt.Sprintf("+ host3 %s:%s", srcIP, srcPort)) return &streamProps{isTapTarget: true, isOutgoing: true} - } else if inArrayString(gSettings.filterAuthorities, srcIP) { + } else if inArrayPod(factory.opts.FilterAuthorities, srcIP) { logger.Log.Debugf("getStreamProps %s", fmt.Sprintf("+ host4 %s", srcIP)) return &streamProps{isTapTarget: true, isOutgoing: true} } diff --git a/tap/tcp_streams_map.go b/tap/tcp_streams_map.go index 2ea55be8e..7e4489dd7 100644 --- a/tap/tcp_streams_map.go +++ b/tap/tcp_streams_map.go @@ -46,7 +46,7 @@ func (streamMap *tcpStreamMap) closeTimedoutTcpStreamChannels() { if !stream.isClosed && time.Now().After(streamWrapper.createdAt.Add(tcpStreamChannelTimeout)) { stream.Close() diagnose.AppStats.IncDroppedTcpStreams() - logger.Log.Debugf("Dropped an unidentified TCP stream because of timeout. Total dropped: %d Total Goroutines: %d Timeout (ms): %d\n", + logger.Log.Debugf("Dropped an unidentified TCP stream because of timeout. Total dropped: %d Total Goroutines: %d Timeout (ms): %d", diagnose.AppStats.DroppedTcpStreams, runtime.NumGoroutine(), tcpStreamChannelTimeout/1000000) } } else { diff --git a/tap/tester/.gitignore b/tap/tester/.gitignore deleted file mode 100644 index 6028450ae..000000000 --- a/tap/tester/.gitignore +++ /dev/null @@ -1 +0,0 @@ -tester \ No newline at end of file diff --git a/tap/tester/README.md b/tap/tester/README.md deleted file mode 100644 index 3fc7e4c65..000000000 --- a/tap/tester/README.md +++ /dev/null @@ -1,12 +0,0 @@ - -This tester used to launch passive-tapper locally without Docker or Kuberenetes environment. - -Its good for testing purposes. - -# How to run - -From the `tap` folder run: -`./tester/launch.sh` - -The tester gets the same arguments the passive_tapper gets, run with `--help` to get a complete list of options. -`./tester/launch.sh --help` diff --git a/tap/tester/launch.sh b/tap/tester/launch.sh deleted file mode 100755 index 5abab4a93..000000000 --- a/tap/tester/launch.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -e - -echo "Building extensions..." -pushd .. && ./devops/build_extensions.sh && popd - -go build -o tester tester/tester.go - -sudo ./tester/tester "$@" diff --git a/tap/tester/tester.go b/tap/tester/tester.go deleted file mode 100644 index d42d0dbd8..000000000 --- a/tap/tester/tester.go +++ /dev/null @@ -1,114 +0,0 @@ -package main - -import ( - "bufio" - "io/ioutil" - "os" - "path" - "plugin" - "sort" - "strings" - - "github.com/op/go-logging" - - "github.com/go-errors/errors" - "github.com/up9inc/mizu/shared/logger" - "github.com/up9inc/mizu/tap" - tapApi "github.com/up9inc/mizu/tap/api" -) - -func loadExtensions() ([]*tapApi.Extension, error) { - extensionsDir := "./extensions" - files, err := ioutil.ReadDir(extensionsDir) - - if err != nil { - return nil, errors.Wrap(err, 0) - } - - extensions := make([]*tapApi.Extension, 0) - for _, file := range files { - filename := file.Name() - - if !strings.HasSuffix(filename, ".so") { - continue - } - - logger.Log.Infof("Loading extension: %s\n", filename) - - extension := &tapApi.Extension{ - Path: path.Join(extensionsDir, filename), - } - - plug, err := plugin.Open(extension.Path) - - if err != nil { - return nil, errors.Wrap(err, 0) - } - - extension.Plug = plug - symDissector, err := plug.Lookup("Dissector") - - if err != nil { - return nil, errors.Wrap(err, 0) - } - - dissector, ok := symDissector.(tapApi.Dissector) - - if !ok { - return nil, errors.Errorf("Symbol Dissector type error: %v %T\n", file, symDissector) - } - - dissector.Register(extension) - extension.Dissector = dissector - extensions = append(extensions, extension) - } - - sort.Slice(extensions, func(i, j int) bool { - return extensions[i].Protocol.Priority < extensions[j].Protocol.Priority - }) - - for _, extension := range extensions { - logger.Log.Infof("Extension Properties: %+v\n", extension) - } - - return extensions, nil -} - -func internalRun() error { - logger.InitLoggerStderrOnly(logging.DEBUG) - - opts := tap.TapOpts{ - HostMode: false, - } - - outputItems := make(chan *tapApi.OutputChannelItem, 1000) - extenssions, err := loadExtensions() - - if err != nil { - return err - } - - tapOpts := tapApi.TrafficFilteringOptions{} - - tap.StartPassiveTapper(&opts, outputItems, extenssions, &tapOpts) - - logger.Log.Infof("Tapping, press enter to exit...\n") - reader := bufio.NewReader(os.Stdin) - reader.ReadLine() - return nil -} - -func main() { - err := internalRun() - - if err != nil { - switch err := err.(type) { - case *errors.Error: - logger.Log.Errorf("Error: %v\n", err.ErrorStack()) - default: - logger.Log.Errorf("Error: %v\n", err) - } - - os.Exit(1) - } -} diff --git a/ui/.snyk b/ui/.snyk index 1e10665fd..232a8b018 100644 --- a/ui/.snyk +++ b/ui/.snyk @@ -133,3 +133,6 @@ ignore: SNYK-JS-WS-1296835: - '*': reason: Non given + SNYK-JS-JSONSCHEMA-1920922: + - '*': + reason: Non given diff --git a/ui/package-lock.json b/ui/package-lock.json index f6a4eff73..a0fa2a952 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -1769,6 +1769,33 @@ } } }, + "@mapbox/rehype-prism": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@mapbox/rehype-prism/-/rehype-prism-0.7.0.tgz", + "integrity": "sha512-zSG46selA6v+3THhCatTyOt9DuTzxTIVTxTbcj15kFmxPDtjzZ5VoFVCLZfjWFouYa9PiXxcbMLLhJoVzCxh9w==", + "requires": { + "hast-util-to-string": "^1.0.4", + "refractor": "^3.4.0", + "unist-util-visit": "^2.0.3" + }, + "dependencies": { + "prismjs": { + "version": "1.24.1", + "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.24.1.tgz", + "integrity": "sha512-mNPsedLuk90RVJioIky8ANZEwYm5w9LcvCXrxHlwf4fNVSn8jEipMybMkWUyyF0JhnC+C4VcOVSBuHRKs1L5Ow==" + }, + "refractor": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/refractor/-/refractor-3.4.0.tgz", + "integrity": "sha512-dBeD02lC5eytm9Gld2Mx0cMcnR+zhSnsTfPpWqFaMgUMJfC9A6bcN3Br/NaXrnBJcuxnLFR90k1jrkaSyV8umg==", + "requires": { + "hastscript": "^6.0.0", + "parse-entities": "^2.0.0", + "prismjs": "~1.24.0" + } + } + } + }, "@material-ui/core": { "version": "4.12.2", "resolved": "https://registry.npmjs.org/@material-ui/core/-/core-4.12.2.tgz", @@ -1788,6 +1815,14 @@ "react-transition-group": "^4.4.0" } }, + "@material-ui/icons": { + "version": "4.11.2", + "resolved": "https://registry.npmjs.org/@material-ui/icons/-/icons-4.11.2.tgz", + "integrity": "sha512-fQNsKX2TxBmqIGJCSi3tGTO/gZ+eJgWmMJkgDiOfyNaunNaxcklJQFaFogYcFl0qFuaEz1qaXYXboa/bUXVSOQ==", + "requires": { + "@babel/runtime": "^7.4.4" + } + }, "@material-ui/lab": { "version": "4.0.0-alpha.60", "resolved": "https://registry.npmjs.org/@material-ui/lab/-/lab-4.0.0-alpha.60.tgz", @@ -2406,6 +2441,11 @@ "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==" }, + "@types/parse5": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@types/parse5/-/parse5-6.0.1.tgz", + "integrity": "sha512-ARATsLdrGPUnaBvxLhUlnltcMgn7pQG312S8ccdYlnyijabrX9RN/KN/iGj9Am96CoW8e/K9628BA7Bv4XHdrA==" + }, "@types/prettier": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.2.3.tgz", @@ -2629,6 +2669,26 @@ "eslint-visitor-keys": "^2.0.0" } }, + "@uiw/react-textarea-code-editor": { + "version": "1.4.12", + "resolved": "https://registry.npmjs.org/@uiw/react-textarea-code-editor/-/react-textarea-code-editor-1.4.12.tgz", + "integrity": "sha512-op0aIRxX8hLi8OLwm/23dQ2X4o2xHUoK2pLr1DWBFgNbIh4L+RM5zByNFjV8mHfi/NvE6mQLB6LtMd1qaor5MQ==", + "requires": { + "@babel/runtime": "7.15.4", + "@mapbox/rehype-prism": "0.7.0", + "rehype": "12.0.0" + }, + "dependencies": { + "@babel/runtime": { + "version": "7.15.4", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.15.4.tgz", + "integrity": "sha512-99catp6bHCaxr4sJ/DbTGgHS4+Rs2RVd2g7iOap6SLGPDknRK9ztKNsE/Fg6QhSeh1FGE5f6gHGQmvvn3I3xhw==", + "requires": { + "regenerator-runtime": "^0.13.4" + } + } + } + }, "@webassemblyjs/ast": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz", @@ -3689,6 +3749,11 @@ "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.18.0.tgz", "integrity": "sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ==" }, + "bail": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.1.tgz", + "integrity": "sha512-d5FoTAr2S5DSUPKl85WNm2yUwsINN8eidIdIwsOge2t33DaOfOdSmmsI11jMN3GmALCXaw+Y6HMVHDzePshFAA==" + }, "balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -4179,6 +4244,11 @@ "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, + "ccount": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.0.tgz", + "integrity": "sha512-VOR0NWFYX65n9gELQdcpqsie5L5ihBXuZGAgaPEp/U7IOSjnPMEH6geE+2f6lcekaNEfWzAHS45mPvSo5bqsUA==" + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -4199,6 +4269,11 @@ "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.4.tgz", "integrity": "sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw==" }, + "character-entities-html4": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.0.0.tgz", + "integrity": "sha512-dwT2xh5ZhUAjyP96k57ilMKoTQyASaw9IAMR9U5c1lCu2RUni6O6jxfpUEdO2RcPT6TJFvr8pqsbami4Jk+2oA==" + }, "character-entities-legacy": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz", @@ -7534,11 +7609,121 @@ "minimalistic-assert": "^1.0.1" } }, + "hast-util-from-parse5": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-7.1.0.tgz", + "integrity": "sha512-m8yhANIAccpU4K6+121KpPP55sSl9/samzQSQGpb0mTExcNh2WlvjtMwSWFhg6uqD4Rr6Nfa8N6TMypQM51rzQ==", + "requires": { + "@types/hast": "^2.0.0", + "@types/parse5": "^6.0.0", + "@types/unist": "^2.0.0", + "hastscript": "^7.0.0", + "property-information": "^6.0.0", + "vfile": "^5.0.0", + "vfile-location": "^4.0.0", + "web-namespaces": "^2.0.0" + }, + "dependencies": { + "comma-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.2.tgz", + "integrity": "sha512-G5yTt3KQN4Yn7Yk4ed73hlZ1evrFKXeUW3086p3PRFNp7m2vIjI6Pg+Kgb+oyzhd9F2qdcoj67+y3SdxL5XWsg==" + }, + "hast-util-parse-selector": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-3.1.0.tgz", + "integrity": "sha512-AyjlI2pTAZEOeu7GeBPZhROx0RHBnydkQIXlhnFzDi0qfXTmGUWoCYZtomHbrdrheV4VFUlPcfJ6LMF5T6sQzg==", + "requires": { + "@types/hast": "^2.0.0" + } + }, + "hastscript": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-7.0.2.tgz", + "integrity": "sha512-uA8ooUY4ipaBvKcMuPehTAB/YfFLSSzCwFSwT6ltJbocFUKH/GDHLN+tflq7lSRf9H86uOuxOFkh1KgIy3Gg2g==", + "requires": { + "@types/hast": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-parse-selector": "^3.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0" + } + }, + "property-information": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.0.1.tgz", + "integrity": "sha512-F4WUUAF7fMeF4/JUFHNBWDaKDXi2jbvqBW/y6o5wsf3j19wTZ7S60TmtB5HoBhtgw7NKQRMWuz5vk2PR0CygUg==" + }, + "space-separated-tokens": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.1.tgz", + "integrity": "sha512-ekwEbFp5aqSPKaqeY1PGrlGQxPNaq+Cnx4+bE2D8sciBQrHpbwoBbawqTN2+6jPs9IdWxxiUcN0K2pkczD3zmw==" + } + } + }, + "hast-util-is-element": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-2.1.1.tgz", + "integrity": "sha512-ag0fiZfRWsPiR1udvnSbaazJLGv8qd8E+/e3rW8rUZhbKG4HNJmFL4QkEceN+22BgE+uozXY30z/s+2dL6Z++g==", + "requires": { + "@types/hast": "^2.0.0", + "@types/unist": "^2.0.0" + } + }, "hast-util-parse-selector": { "version": "2.2.5", "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz", "integrity": "sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ==" }, + "hast-util-to-html": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-8.0.2.tgz", + "integrity": "sha512-ipLhUTMyyJi9F/LXaNDG9BrRdshP6obCfmUZYbE/+T639IdzqAOkKN4DyrEyID0gbb+rsC3PKf0XlviZwzomhw==", + "requires": { + "@types/hast": "^2.0.0", + "ccount": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-is-element": "^2.0.0", + "hast-util-whitespace": "^2.0.0", + "html-void-elements": "^2.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "stringify-entities": "^4.0.0", + "unist-util-is": "^5.0.0" + }, + "dependencies": { + "comma-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.2.tgz", + "integrity": "sha512-G5yTt3KQN4Yn7Yk4ed73hlZ1evrFKXeUW3086p3PRFNp7m2vIjI6Pg+Kgb+oyzhd9F2qdcoj67+y3SdxL5XWsg==" + }, + "property-information": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.0.1.tgz", + "integrity": "sha512-F4WUUAF7fMeF4/JUFHNBWDaKDXi2jbvqBW/y6o5wsf3j19wTZ7S60TmtB5HoBhtgw7NKQRMWuz5vk2PR0CygUg==" + }, + "space-separated-tokens": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.1.tgz", + "integrity": "sha512-ekwEbFp5aqSPKaqeY1PGrlGQxPNaq+Cnx4+bE2D8sciBQrHpbwoBbawqTN2+6jPs9IdWxxiUcN0K2pkczD3zmw==" + }, + "unist-util-is": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-5.1.1.tgz", + "integrity": "sha512-F5CZ68eYzuSvJjGhCLPL3cYx45IxkqXSetCcRgUXtbcm50X2L9oOWQlfUfDdAf+6Pd27YDblBfdtmsThXmwpbQ==" + } + } + }, + "hast-util-to-string": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/hast-util-to-string/-/hast-util-to-string-1.0.4.tgz", + "integrity": "sha512-eK0MxRX47AV2eZ+Lyr18DCpQgodvaS3fAQO2+b9Two9F5HEoRPhiUMNzoXArMJfZi2yieFzUBMRl3HNJ3Jus3w==" + }, + "hast-util-whitespace": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-2.0.0.tgz", + "integrity": "sha512-Pkw+xBHuV6xFeJprJe2BBEoDV+AvQySaz3pPDRUs5PNZEMQjpXJJueqrpcHIXxnWTcAGi/UOCgVShlkY6kLoqg==" + }, "hastscript": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-6.0.0.tgz", @@ -7671,6 +7856,11 @@ "terser": "^4.6.3" } }, + "html-void-elements": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-2.0.0.tgz", + "integrity": "sha512-4OYzQQsBt0G9bJ/nM9/DDsjm4+fVdzAaPJJcWk5QwA3GIAPxQEeOR0rsI8HbDHQz5Gta8pVvGnnTNSbZVEVvkQ==" + }, "html-webpack-plugin": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-4.5.0.tgz", @@ -10890,6 +11080,11 @@ "minimist": "^1.2.5" } }, + "moment": { + "version": "2.29.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", + "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" + }, "move-concurrently": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz", @@ -13454,9 +13649,9 @@ } }, "react-scrollable-feed-virtualized": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/react-scrollable-feed-virtualized/-/react-scrollable-feed-virtualized-1.4.3.tgz", - "integrity": "sha512-M9WgJKr57jCyWKNCksc3oi+xhtO0YbL9d7Ll8Sdc5ZWOIstNvdNbNX0k4Nq6kXUVaHCJ9qE8omdSI/CxT3MLAQ==" + "version": "1.4.9", + "resolved": "https://registry.npmjs.org/react-scrollable-feed-virtualized/-/react-scrollable-feed-virtualized-1.4.9.tgz", + "integrity": "sha512-YkFkPjdIXDUsaCNYhZ+Blpp3LF+CsJWscwn/0fGSjF5QBKCtPURO9AEUA362Qnjr4S8LF2IjSAOCCFedIEnVNw==" }, "react-syntax-highlighter": { "version": "15.4.3", @@ -13470,6 +13665,14 @@ "refractor": "^3.2.0" } }, + "react-toastify": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/react-toastify/-/react-toastify-8.0.3.tgz", + "integrity": "sha512-rv3koC7f9lKKSkdpYgo/TGzgWlrB/aaiUInF1DyV7BpiM4kyTs+uhu6/r8XDMtBY2FOIHK+FlK3Iv7OzpA/tCA==", + "requires": { + "clsx": "^1.1.1" + } + }, "react-transition-group": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.1.tgz", @@ -13693,6 +13896,38 @@ } } }, + "rehype": { + "version": "12.0.0", + "resolved": "https://registry.npmjs.org/rehype/-/rehype-12.0.0.tgz", + "integrity": "sha512-gZcttmf9R5IYHb8AlI1rlmWqXS1yX0rSB/S5ZGJs8atfYZy2DobvH3Ic/gSzB+HL/+oOHPtBguw1TprfhxXBgQ==", + "requires": { + "@types/hast": "^2.0.0", + "rehype-parse": "^8.0.0", + "rehype-stringify": "^9.0.0", + "unified": "^10.0.0" + } + }, + "rehype-parse": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/rehype-parse/-/rehype-parse-8.0.3.tgz", + "integrity": "sha512-RGw0CVt+0S6KdvpE8bbP2Db9WXclQcIX7A0ufM3QFqAhTo/ddJMQrrI2j3cijlRPZlGK8R3pRgC8U5HyV76IDw==", + "requires": { + "@types/hast": "^2.0.0", + "hast-util-from-parse5": "^7.0.0", + "parse5": "^6.0.0", + "unified": "^10.0.0" + } + }, + "rehype-stringify": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/rehype-stringify/-/rehype-stringify-9.0.2.tgz", + "integrity": "sha512-BuVA6lAEYtOpXO2xuHLohAzz8UNoQAxAqYRqh4QEEtU39Co+P1JBZhw6wXA9hMWp+JLcmrxWH8+UKcNSr443Fw==", + "requires": { + "@types/hast": "^2.0.0", + "hast-util-to-html": "^8.0.0", + "unified": "^10.0.0" + } + }, "relateurl": { "version": "0.2.7", "resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz", @@ -15342,6 +15577,22 @@ } } }, + "stringify-entities": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.1.tgz", + "integrity": "sha512-gmMQxKXPWIO3NXNSPyWNhlYcBNGpPA/487D+9dLPnU4xBnIrnHdr8cv5rGJOS/1BRxEXRb7uKwg7BA36IWV7xg==", + "requires": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^2.0.0" + }, + "dependencies": { + "character-entities-legacy": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-2.0.0.tgz", + "integrity": "sha512-YwaEtEvWLpFa6Wh3uVLrvirA/ahr9fki/NUd/Bd4OR6EdJ8D22hovYQEOUCBfQfcqnC4IAMGMsHXY1eXgL4ZZA==" + } + } + }, "stringify-object": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz", @@ -15877,6 +16128,11 @@ "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz", "integrity": "sha1-WIeWa7WCpFA6QetST301ARgVphM=" }, + "trough": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.0.2.tgz", + "integrity": "sha512-FnHq5sTMxC0sk957wHDzRnemFnNBvt/gSY99HzK8F7UP5WAbvP70yX5bd7CjEQkN+TjdxwI7g7lJ6podqrG2/w==" + }, "true-case-path": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/true-case-path/-/true-case-path-1.0.3.tgz", @@ -16037,6 +16293,32 @@ "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-1.1.0.tgz", "integrity": "sha512-PqSoPh/pWetQ2phoj5RLiaqIk4kCNwoV3CI+LfGmWLKI3rE3kl1h59XpX2BjgDrmbxD9ARtQobPGU1SguCYuQg==" }, + "unified": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/unified/-/unified-10.1.0.tgz", + "integrity": "sha512-4U3ru/BRXYYhKbwXV6lU6bufLikoAavTwev89H5UxY8enDFaAT2VXmIXYNm6hb5oHPng/EXr77PVyDFcptbk5g==", + "requires": { + "@types/unist": "^2.0.0", + "bail": "^2.0.0", + "extend": "^3.0.0", + "is-buffer": "^2.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^5.0.0" + }, + "dependencies": { + "is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" + }, + "is-plain-obj": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.0.0.tgz", + "integrity": "sha512-NXRbBtUdBioI73y/HmOhogw/U5msYPC9DAtGkJXeFcFWSFZw0mCUsPxk/snTuJHzNKA8kLBK4rH97RMB1BfCXw==" + } + } + }, "union-value": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", @@ -16082,6 +16364,38 @@ "crypto-random-string": "^1.0.0" } }, + "unist-util-is": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz", + "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==" + }, + "unist-util-stringify-position": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-3.0.0.tgz", + "integrity": "sha512-SdfAl8fsDclywZpfMDTVDxA2V7LjtRDTOFd44wUJamgl6OlVngsqWjxvermMYf60elWHbxhuRCZml7AnuXCaSA==", + "requires": { + "@types/unist": "^2.0.0" + } + }, + "unist-util-visit": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-2.0.3.tgz", + "integrity": "sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0", + "unist-util-visit-parents": "^3.0.0" + } + }, + "unist-util-visit-parents": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz", + "integrity": "sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0" + } + }, "universalify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", @@ -16306,6 +16620,42 @@ "extsprintf": "^1.2.0" } }, + "vfile": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-5.1.0.tgz", + "integrity": "sha512-4o7/DJjEaFPYSh0ckv5kcYkJTHQgCKdL8ozMM1jLAxO9ox95IzveDPXCZp08HamdWq8JXTkClDvfAKaeLQeKtg==", + "requires": { + "@types/unist": "^2.0.0", + "is-buffer": "^2.0.0", + "unist-util-stringify-position": "^3.0.0", + "vfile-message": "^3.0.0" + }, + "dependencies": { + "is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" + } + } + }, + "vfile-location": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-4.0.1.tgz", + "integrity": "sha512-JDxPlTbZrZCQXogGheBHjbRWjESSPEak770XwWPfw5mTc1v1nWGLB/apzZxsx8a0SJVfF8HK8ql8RD308vXRUw==", + "requires": { + "@types/unist": "^2.0.0", + "vfile": "^5.0.0" + } + }, + "vfile-message": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-3.0.2.tgz", + "integrity": "sha512-UUjZYIOg9lDRwwiBAuezLIsu9KlXntdxwG+nXnjuQAHvBpcX3x0eN8h+I7TkY5nkCXj+cWVp4ZqebtGBvok8ww==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-stringify-position": "^3.0.0" + } + }, "vm-browserify": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", @@ -16590,6 +16940,11 @@ "minimalistic-assert": "^1.0.0" } }, + "web-namespaces": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.0.tgz", + "integrity": "sha512-dE7ELZRVWh0ceQsRgkjLgsAvwTuv3kcjSY/hLjqL0llleUlQBDjE9JkB9FCBY5F2mnFEwiyJoowl8+NVGHe8dw==" + }, "web-vitals": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/web-vitals/-/web-vitals-1.1.1.tgz", diff --git a/ui/package.json b/ui/package.json index 463ef8c4b..f8fcebd2a 100644 --- a/ui/package.json +++ b/ui/package.json @@ -4,6 +4,7 @@ "private": true, "dependencies": { "@material-ui/core": "^4.11.3", + "@material-ui/icons": "^4.11.2", "@material-ui/lab": "^4.0.0-alpha.60", "@testing-library/jest-dom": "^5.11.10", "@testing-library/react": "^11.2.6", @@ -12,8 +13,10 @@ "@types/node": "^12.20.10", "@types/react": "^17.0.3", "@types/react-dom": "^17.0.3", + "@uiw/react-textarea-code-editor": "^1.4.12", "axios": "^0.21.1", "jsonpath": "^1.1.1", + "moment": "^2.29.1", "node-sass": "^5.0.0", "numeral": "^2.0.6", "protobuf-decoder": "^0.1.0", @@ -21,8 +24,9 @@ "react-copy-to-clipboard": "^5.0.3", "react-dom": "^17.0.2", "react-scripts": "4.0.3", - "react-scrollable-feed-virtualized": "^1.4.3", + "react-scrollable-feed-virtualized": "^1.4.9", "react-syntax-highlighter": "^15.4.3", + "react-toastify": "^8.0.3", "typescript": "^4.2.4", "web-vitals": "^1.1.1" }, diff --git a/ui/src/components/EntriesList.tsx b/ui/src/components/EntriesList.tsx index 53e653a05..6b3a6917b 100644 --- a/ui/src/components/EntriesList.tsx +++ b/ui/src/components/EntriesList.tsx @@ -1,43 +1,45 @@ -import {EntryItem} from "./EntryListItem/EntryListItem"; import React, {useCallback, useEffect, useMemo, useRef, useState} from "react"; import styles from './style/EntriesList.module.sass'; -import spinner from './assets/spinner.svg'; import ScrollableFeedVirtualized from "react-scrollable-feed-virtualized"; -import {StatusType} from "./Filters"; -import Api from "../helpers/api"; +import Moment from 'moment'; +import {EntryItem} from "./EntryListItem/EntryListItem"; import down from "./assets/downImg.svg"; +import spinner from './assets/spinner.svg'; +import Api from "../helpers/api"; interface EntriesListProps { entries: any[]; - setEntries: (entries: any[]) => void; - focusedEntryId: string; - setFocusedEntryId: (id: string) => void; - connectionOpen: boolean; + setEntries: any; + query: string; + listEntryREF: any; + onSnapBrokenEvent: () => void; + isSnappedToBottom: boolean; + setIsSnappedToBottom: any; + queriedCurrent: number; + setQueriedCurrent: any; + queriedTotal: number; + setQueriedTotal: any; + startTime: number; noMoreDataTop: boolean; setNoMoreDataTop: (flag: boolean) => void; - noMoreDataBottom: boolean; - setNoMoreDataBottom: (flag: boolean) => void; - methodsFilter: Array; - statusFilter: Array; - pathFilter: string; - serviceFilter: string; - listEntryREF: any; - onScrollEvent: (isAtBottom:boolean) => void; - scrollableList: boolean; -} - -enum FetchOperator { - LT = "lt", - GT = "gt" + focusedEntryId: string; + setFocusedEntryId: (id: string) => void; + updateQuery: any; + leftOffTop: number; + setLeftOffTop: (leftOffTop: number) => void; + isWebSocketConnectionClosed: boolean; + ws: any; + openWebSocket: (query: string, resetEntries: boolean) => void; + leftOffBottom: number; + truncatedTimestamp: number; + setTruncatedTimestamp: any; } const api = new Api(); -export const EntriesList: React.FC = ({entries, setEntries, focusedEntryId, setFocusedEntryId, connectionOpen, noMoreDataTop, setNoMoreDataTop, noMoreDataBottom, setNoMoreDataBottom, methodsFilter, statusFilter, pathFilter, serviceFilter, listEntryREF, onScrollEvent, scrollableList}) => { - +export const EntriesList: React.FC = ({entries, setEntries, query, listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, setQueriedCurrent, queriedTotal, setQueriedTotal, startTime, noMoreDataTop, setNoMoreDataTop, focusedEntryId, setFocusedEntryId, updateQuery, leftOffTop, setLeftOffTop, isWebSocketConnectionClosed, ws, openWebSocket, leftOffBottom, truncatedTimestamp, setTruncatedTimestamp}) => { const [loadMoreTop, setLoadMoreTop] = useState(false); const [isLoadingTop, setIsLoadingTop] = useState(false); - const scrollableRef = useRef(null); useEffect(() => { @@ -47,63 +49,57 @@ export const EntriesList: React.FC = ({entries, setEntries, fo if(el.scrollTop === 0) { setLoadMoreTop(true); } else { + setNoMoreDataTop(false); setLoadMoreTop(false); } }); - }, []); + }, [setLoadMoreTop, setNoMoreDataTop]); - const filterEntries = useCallback((entry) => { - if(methodsFilter.length > 0 && !methodsFilter.includes(entry.method.toLowerCase())) return; - if(pathFilter && entry.path?.toLowerCase()?.indexOf(pathFilter) === -1) return; - if(serviceFilter && entry.service?.toLowerCase()?.indexOf(serviceFilter) === -1) return; - if(statusFilter.includes(StatusType.SUCCESS) && entry.statusCode >= 400) return; - if(statusFilter.includes(StatusType.ERROR) && entry.statusCode < 400) return; - return entry; - },[methodsFilter, pathFilter, statusFilter, serviceFilter]) - - const filteredEntries = useMemo(() => { - return entries.filter(filterEntries); - },[entries, filterEntries]) + const memoizedEntries = useMemo(() => { + return entries; + },[entries]); const getOldEntries = useCallback(async () => { - setIsLoadingTop(true); - const data = await api.fetchEntries(FetchOperator.LT, entries[0].timestamp); setLoadMoreTop(false); - - let scrollTo; - if(data.length === 0) { + if (leftOffTop === null || leftOffTop <= 0) { + return; + } + setIsLoadingTop(true); + const data = await api.fetchEntries(leftOffTop, -1, query, 100, 3000); + if (!data || data.data === null || data.meta === null) { setNoMoreDataTop(true); - scrollTo = document.getElementById("noMoreDataTop"); + setIsLoadingTop(false); + return; + } + setLeftOffTop(data.meta.leftOff); + + let scrollTo: boolean; + if (data.meta.leftOff === 0) { + setNoMoreDataTop(true); + scrollTo = false; } else { - scrollTo = document.getElementById(filteredEntries?.[0]?.id); + scrollTo = true; } setIsLoadingTop(false); - const newEntries = [...data, ...entries]; + + const newEntries = [...data.data.reverse(), ...entries]; setEntries(newEntries); - if(scrollTo) { - scrollTo.scrollIntoView(); + setQueriedCurrent(queriedCurrent + data.meta.current); + setQueriedTotal(data.meta.total); + setTruncatedTimestamp(data.meta.truncatedTimestamp); + + if (scrollTo) { + scrollableRef.current.scrollToIndex(data.data.length - 1); } - },[setLoadMoreTop, setIsLoadingTop, entries, setEntries, filteredEntries, setNoMoreDataTop]) + },[setLoadMoreTop, setIsLoadingTop, entries, setEntries, query, setNoMoreDataTop, leftOffTop, setLeftOffTop, queriedCurrent, setQueriedCurrent, setQueriedTotal, setTruncatedTimestamp]); useEffect(() => { - if(!loadMoreTop || connectionOpen || noMoreDataTop) return; + if(!isWebSocketConnectionClosed || !loadMoreTop || noMoreDataTop) return; getOldEntries(); - }, [loadMoreTop, connectionOpen, noMoreDataTop, getOldEntries]); + }, [loadMoreTop, noMoreDataTop, getOldEntries, isWebSocketConnectionClosed]); - const getNewEntries = async () => { - const data = await api.fetchEntries(FetchOperator.GT, entries[entries.length - 1].timestamp); - let scrollTo; - if(data.length === 0) { - setNoMoreDataBottom(true); - } - scrollTo = document.getElementById(filteredEntries?.[filteredEntries.length -1]?.id); - let newEntries = [...entries, ...data]; - setEntries(newEntries); - if(scrollTo) { - scrollTo.scrollIntoView({behavior: "smooth"}); - } - } + const scrollbarVisible = scrollableRef.current?.childWrapperRef.current.clientHeight > scrollableRef.current?.wrapperRef.current.clientHeight; return <>
@@ -111,28 +107,50 @@ export const EntriesList: React.FC = ({entries, setEntries, fo {isLoadingTop &&
spinner
} - onScrollEvent(isAtBottom)}> - {noMoreDataTop && !connectionOpen &&
No more data available
} - {filteredEntries.map(entry => )} + {noMoreDataTop &&
No more data available
} + + {false /* It's because the first child is ignored by ScrollableFeedVirtualized */} + {memoizedEntries.map(entry => )} - {!connectionOpen && !noMoreDataBottom &&
-
getNewEntries()}>Fetch more entries
-
} +
- {entries?.length > 0 &&
-
{filteredEntries?.length !== entries.length && `${filteredEntries?.length} / `} {entries?.length} requests
-
Started listening at {new Date(+entries[0].timestamp)?.toLocaleString()}
-
} +
+
Displaying {entries?.length} results out of {queriedTotal} total
+ {startTime !== 0 &&
Started listening at {Moment(truncatedTimestamp ? truncatedTimestamp : startTime).utc().format('MM/DD/YYYY, h:mm:ss.SSS A')}
} +
; }; diff --git a/ui/src/components/EntryDetailed.tsx b/ui/src/components/EntryDetailed.tsx index ffc5151a9..72f8ba5c7 100644 --- a/ui/src/components/EntryDetailed.tsx +++ b/ui/src/components/EntryDetailed.tsx @@ -1,9 +1,9 @@ import React from "react"; import EntryViewer from "./EntryDetailed/EntryViewer"; +import {EntryItem} from "./EntryListItem/EntryListItem"; import {makeStyles} from "@material-ui/core"; import Protocol from "./UI/Protocol" -import StatusCode from "./UI/StatusCode"; -import {EndpointPath} from "./UI/EndpointPath"; +import Queryable from "./UI/Queryable"; const useStyles = makeStyles(() => ({ entryTitle: { @@ -12,6 +12,7 @@ const useStyles = makeStyles(() => ({ maxHeight: 46, alignItems: 'center', marginBottom: 4, + marginLeft: 6, padding: 2, paddingBottom: 0 }, @@ -28,50 +29,83 @@ const useStyles = makeStyles(() => ({ interface EntryDetailedProps { entryData: any + updateQuery: any } export const formatSize = (n: number) => n > 1000 ? `${Math.round(n / 1000)}KB` : `${n} B`; -const EntryTitle: React.FC = ({protocol, data, bodySize, elapsedTime}) => { +const EntryTitle: React.FC = ({protocol, data, bodySize, elapsedTime, updateQuery}) => { const classes = useStyles(); - const {response} = JSON.parse(data.entry); - + const response = data.response; return
- +
- {response.payload &&
{formatSize(bodySize)}
} - {response.payload &&
{Math.round(elapsedTime)}ms
} + {response && +
+ {formatSize(bodySize)} +
+
} + {response && = ${elapsedTime}`} + updateQuery={updateQuery} + style={{marginRight: 18}} + displayIconOnMouseOver={true} + > +
+ {Math.round(elapsedTime)}ms +
+
}
; }; -const EntrySummary: React.FC = ({data}) => { - const classes = useStyles(); +const EntrySummary: React.FC = ({data, updateQuery}) => { + const entry = data.base; - const {response, request} = JSON.parse(data.entry); - - return
- {response?.payload && response.payload?.details && "status" in response.payload.details &&
- -
} -
- -
-
; + return ; }; -export const EntryDetailed: React.FC = ({entryData}) => { +export const EntryDetailed: React.FC = ({entryData, updateQuery}) => { return <> - {entryData.data && } + {entryData.data && } <> - {entryData.data && } + {entryData.data && } }; diff --git a/ui/src/components/EntryDetailed/EntrySections.module.sass b/ui/src/components/EntryDetailed/EntrySections.module.sass index a7ec762ca..b1b8471f6 100644 --- a/ui/src/components/EntryDetailed/EntrySections.module.sass +++ b/ui/src/components/EntryDetailed/EntrySections.module.sass @@ -27,7 +27,7 @@ font-weight: 600 font-size: .75rem line-height: 1.2 - margin: .3rem 0 + margin-bottom: -2px .dataKey color: $blue-gray diff --git a/ui/src/components/EntryDetailed/EntrySections.tsx b/ui/src/components/EntryDetailed/EntrySections.tsx index f364b4a51..7201b5206 100644 --- a/ui/src/components/EntryDetailed/EntrySections.tsx +++ b/ui/src/components/EntryDetailed/EntrySections.tsx @@ -3,17 +3,42 @@ import React, {useState} from "react"; import {SyntaxHighlighter} from "../UI/SyntaxHighlighter/index"; import CollapsibleContainer from "../UI/CollapsibleContainer"; import FancyTextDisplay from "../UI/FancyTextDisplay"; +import Queryable from "../UI/Queryable"; import Checkbox from "../UI/Checkbox"; import ProtobufDecoder from "protobuf-decoder"; interface EntryViewLineProps { label: string; value: number | string; + updateQuery: any; + selector: string; + overrideQueryValue?: string; } -const EntryViewLine: React.FC = ({label, value}) => { - return (label && value && - {label} +const EntryViewLine: React.FC = ({label, value, updateQuery, selector, overrideQueryValue}) => { + let query: string; + if (!selector) { + query = ""; + } else if (overrideQueryValue) { + query = `${selector} == ${overrideQueryValue}`; + } else if (typeof(value) == "string") { + query = `${selector} == "${JSON.stringify(value).slice(1, -1)}"`; + } else { + query = `${selector} == ${value}`; + } + return (label && + + + {label} + + = ({title, color, isExpanded}) => { return
- +
{isExpanded ? '-' : '+'} - +
{title}
} @@ -62,15 +87,19 @@ export const EntrySectionContainer: React.FC = ({tit interface EntryBodySectionProps { content: any, color: string, + updateQuery: any, encoding?: string, contentType?: string, + selector?: string, } export const EntryBodySection: React.FC = ({ color, + updateQuery, content, encoding, contentType, + selector, }) => { const MAXIMUM_BYTES_TO_HIGHLIGHT = 10000; // The maximum of chars to highlight in body, in case the response can be megabytes const supportedLanguages = [['html', 'html'], ['json', 'json'], ['application/grpc', 'json']]; // [[indicator, languageToUse],...] @@ -107,8 +136,8 @@ export const EntryBodySection: React.FC = ({ {content && content?.length > 0 && - - + + {encoding && }
@@ -132,17 +161,23 @@ interface EntrySectionProps { title: string, color: string, arrayToIterate: any[], + updateQuery: any, } -export const EntryTableSection: React.FC = ({title, color, arrayToIterate}) => { +export const EntryTableSection: React.FC = ({title, color, arrayToIterate, updateQuery}) => { return { arrayToIterate && arrayToIterate.length > 0 ? - {arrayToIterate.map(({name, value}, index) => )} + {arrayToIterate.map(({name, value, selector}, index) => )}
: diff --git a/ui/src/components/EntryDetailed/EntryViewer.module.sass b/ui/src/components/EntryDetailed/EntryViewer.module.sass index 740a4c417..fdc1e7195 100644 --- a/ui/src/components/EntryDetailed/EntryViewer.module.sass +++ b/ui/src/components/EntryDetailed/EntryViewer.module.sass @@ -4,6 +4,7 @@ font-family: "Source Sans Pro", Lucida Grande, Tahoma, sans-serif height: calc(100% - 70px) width: 100% + margin-top: 10px h3, h4 diff --git a/ui/src/components/EntryDetailed/EntryViewer.tsx b/ui/src/components/EntryDetailed/EntryViewer.tsx index dc8b8f4c7..dabf3d5b7 100644 --- a/ui/src/components/EntryDetailed/EntryViewer.tsx +++ b/ui/src/components/EntryDetailed/EntryViewer.tsx @@ -8,7 +8,7 @@ enum SectionTypes { SectionBody = "body", } -const SectionsRepresentation: React.FC = ({data, color}) => { +const SectionsRepresentation: React.FC = ({data, color, updateQuery}) => { const sections = [] if (data) { @@ -16,12 +16,12 @@ const SectionsRepresentation: React.FC = ({data, color}) => { switch (row.type) { case SectionTypes.SectionTable: sections.push( - + ) break; case SectionTypes.SectionBody: sections.push( - + ) break; default: @@ -33,7 +33,7 @@ const SectionsRepresentation: React.FC = ({data, color}) => { return <>{sections}; } -const AutoRepresentation: React.FC = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color}) => { +const AutoRepresentation: React.FC = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color, updateQuery}) => { var TABS = [ { tab: 'Request' @@ -85,10 +85,10 @@ const AutoRepresentation: React.FC = ({representation, isRulesEnabled, rule {currentTab === TABS[0].tab && - + } {response && currentTab === TABS[responseTabIndex].tab && - + } {isRulesEnabled && currentTab === TABS[rulesTabIndex].tab && @@ -110,9 +110,10 @@ interface Props { contractContent: string; color: string; elapsedTime: number; + updateQuery: any; } -const EntryViewer: React.FC = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color}) => { +const EntryViewer: React.FC = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color, updateQuery}) => { return = ({representation, isRulesEnabled, rulesMatc contractContent={contractContent} elapsedTime={elapsedTime} color={color} + updateQuery={updateQuery} /> }; diff --git a/ui/src/components/EntryListItem/EntryListItem.module.sass b/ui/src/components/EntryListItem/EntryListItem.module.sass index d8e5295d5..7e396d136 100644 --- a/ui/src/components/EntryListItem/EntryListItem.module.sass +++ b/ui/src/components/EntryListItem/EntryListItem.module.sass @@ -19,7 +19,6 @@ .rowSelected border: 1px $blue-color solid - margin-right: 3px .ruleSuccessRow background: #E8FFF1 @@ -46,13 +45,12 @@ .ruleNumberTextSuccess color: #219653 -.service +.resolvedName text-overflow: ellipsis overflow: hidden white-space: nowrap color: $secondary-font-color padding-left: 4px - padding-top: 3px padding-right: 10px display: flex font-size: 12px @@ -62,7 +60,7 @@ color: $secondary-font-color padding-left: 12px flex-shrink: 0 - width: 145px + width: 185px text-align: left .endpointServiceContainer @@ -70,7 +68,6 @@ flex-direction: column overflow: hidden padding-right: 10px - padding-left: 10px flex-grow: 1 .separatorRight @@ -84,7 +81,14 @@ padding: 4px padding-left: 12px -.port +.tcpInfo font-size: 12px color: $secondary-font-color - margin: 5px + margin-top: 5px + margin-bottom: 5px + +.port + margin-right: 5px + +.ip + margin-left: 5px diff --git a/ui/src/components/EntryListItem/EntryListItem.tsx b/ui/src/components/EntryListItem/EntryListItem.tsx index fc418631f..d88575250 100644 --- a/ui/src/components/EntryListItem/EntryListItem.tsx +++ b/ui/src/components/EntryListItem/EntryListItem.tsx @@ -1,8 +1,11 @@ import React from "react"; +import Moment from 'moment'; +import SwapHorizIcon from '@material-ui/icons/SwapHoriz'; import styles from './EntryListItem.module.sass'; import StatusCode, {getClassification, StatusCodeClassification} from "../UI/StatusCode"; import Protocol, {ProtocolInterface} from "../UI/Protocol" -import {EndpointPath} from "../UI/EndpointPath"; +import {Summary} from "../UI/Summary"; +import Queryable from "../UI/Queryable"; import ingoingIconSuccess from "../assets/ingoing-traffic-success.svg" import ingoingIconFailure from "../assets/ingoing-traffic-failure.svg" import ingoingIconNeutral from "../assets/ingoing-traffic-neutral.svg" @@ -10,19 +13,21 @@ import outgoingIconSuccess from "../assets/outgoing-traffic-success.svg" import outgoingIconFailure from "../assets/outgoing-traffic-failure.svg" import outgoingIconNeutral from "../assets/outgoing-traffic-neutral.svg" +interface TCPInterface { + ip: string + port: string + name: string +} + interface Entry { protocol: ProtocolInterface, method?: string, summary: string, - service: string, - id: string, + id: number, statusCode?: number; - url?: string; timestamp: Date; - sourceIp: string, - sourcePort: string, - destinationIp: string, - destinationPort: string, + src: TCPInterface, + dst: TCPInterface, isOutgoing?: boolean; latency: number; rules: Rules; @@ -37,12 +42,17 @@ interface Rules { interface EntryProps { entry: Entry; + focusedEntryId: string; setFocusedEntryId: (id: string) => void; - isSelected?: boolean; style: object; + updateQuery: any; + headingMode: boolean; } -export const EntryItem: React.FC = ({entry, setFocusedEntryId, isSelected, style}) => { +export const EntryItem: React.FC = ({entry, focusedEntryId, setFocusedEntryId, style, updateQuery, headingMode}) => { + + const isSelected = focusedEntryId === entry.id.toString(); + const classification = getClassification(entry.statusCode) const numberOfRules = entry.rules.numberOfRules let ingoingIcon; @@ -113,28 +123,66 @@ export const EntryItem: React.FC = ({entry, setFocusedEntryId, isSel break; } + const isStatusCodeEnabled = ((entry.protocol.name === "http" && "statusCode" in entry) || entry.statusCode !== 0); + var endpointServiceContainer = "10px"; + if (!isStatusCodeEnabled) endpointServiceContainer = "20px"; + return <>
setFocusedEntryId(entry.id)} + onClick={() => { + if (!setFocusedEntryId) return; + setFocusedEntryId(entry.id.toString()); + }} style={{ border: isSelected ? `1px ${entry.protocol.backgroundColor} solid` : "1px transparent solid", - position: "absolute", + position: !headingMode ? "absolute" : "unset", top: style['top'], - marginTop: style['marginTop'], - width: "calc(100% - 25px)", + marginTop: !headingMode ? style['marginTop'] : "10px", + width: !headingMode ? "calc(100% - 25px)" : "calc(100% - 18px)", }} > - - {((entry.protocol.name === "http" && "statusCode" in entry) || entry.statusCode !== 0) &&
- + {!headingMode ? : null} + {isStatusCodeEnabled &&
+
} -
- -
- {entry.service} +
+ +
+ + + {entry.src.name ? entry.src.name : "[Unresolved]"} + + + + + + {entry.dst.name ? entry.dst.name : "[Unresolved]"} + +
{ @@ -152,18 +200,109 @@ export const EntryItem: React.FC = ({entry, setFocusedEntryId, isSel : "" }
- {entry.sourcePort} + + + {entry.src.ip} + + + : + + + {entry.src.port} + + {entry.isOutgoing ? - Ingoing traffic + + Ingoing traffic + : - Outgoing traffic + + Outgoing traffic { + updateQuery(`outgoing == false`) + }} + /> + } - {entry.destinationPort} + + + {entry.dst.ip} + + + : + + + {entry.dst.port} + +
- - {new Date(+entry.timestamp)?.toLocaleString()} - + = datetime("${Moment(+entry.timestamp)?.utc().format('MM/DD/YYYY, h:mm:ss.SSS A')}")`} + updateQuery={updateQuery} + displayIconOnMouseOver={true} + flipped={false} + > + + {Moment(+entry.timestamp)?.utc().format('MM/DD/YYYY, h:mm:ss.SSS A')} + +
diff --git a/ui/src/components/Filters.tsx b/ui/src/components/Filters.tsx index c1795a67a..0afc7325d 100644 --- a/ui/src/components/Filters.tsx +++ b/ui/src/components/Filters.tsx @@ -1,137 +1,330 @@ -import React from "react"; +import React, {useRef, useState} from "react"; import styles from './style/Filters.module.sass'; -import {FilterSelect} from "./UI/FilterSelect"; -import {TextField} from "@material-ui/core"; -import {ALL_KEY} from "./UI/Select"; +import {Button, Grid, Modal, Box, Typography, Backdrop, Fade, Divider} from "@material-ui/core"; +import CodeEditor from '@uiw/react-textarea-code-editor'; +import MenuBookIcon from '@material-ui/icons/MenuBook'; +import {SyntaxHighlighter} from "./UI/SyntaxHighlighter/index"; +import filterUIExample1 from "./assets/filter-ui-example-1.png" +import filterUIExample2 from "./assets/filter-ui-example-2.png" +import variables from '../variables.module.scss'; interface FiltersProps { - methodsFilter: Array; - setMethodsFilter: (methods: Array) => void; - statusFilter: Array; - setStatusFilter: (methods: Array) => void; - pathFilter: string - setPathFilter: (val: string) => void; - serviceFilter: string - setServiceFilter: (val: string) => void; + query: string + setQuery: any + backgroundColor: string + ws: any + openWebSocket: (query: string, resetEntries: boolean) => void; } -export const Filters: React.FC = ({methodsFilter, setMethodsFilter, statusFilter, setStatusFilter, pathFilter, setPathFilter, serviceFilter, setServiceFilter}) => { - +export const Filters: React.FC = ({query, setQuery, backgroundColor, ws, openWebSocket}) => { return
- - - - +
; }; -const _toUpperCase = v => v.toUpperCase(); +interface QueryFormProps { + query: string + setQuery: any + backgroundColor: string + ws: any + openWebSocket: (query: string, resetEntries: boolean) => void; +} -const FilterContainer: React.FC = ({children}) => { - return
- {children} -
; +const style = { + position: 'absolute', + top: '10%', + left: '50%', + transform: 'translate(-50%, 0%)', + width: '80vw', + bgcolor: 'background.paper', + borderRadius: '5px', + boxShadow: 24, + p: 4, + color: '#000', }; -enum HTTPMethod { - GET = "get", - PUT = "put", - POST = "post", - DELETE = "delete", - OPTIONS="options", - PATCH = "patch" -} +export const QueryForm: React.FC = ({query, setQuery, backgroundColor, ws, openWebSocket}) => { -interface MethodFilterProps { - methodsFilter: Array; - setMethodsFilter: (methods: Array) => void; -} + const formRef = useRef(null); -const MethodFilter: React.FC = ({methodsFilter, setMethodsFilter}) => { + const [openModal, setOpenModal] = useState(false); - const methodClicked = (val) => { - if(val === ALL_KEY) { - setMethodsFilter([]); - return; - } - if(methodsFilter.includes(val)) { - setMethodsFilter(methodsFilter.filter(method => method !== val)) + const handleOpenModal = () => setOpenModal(true); + const handleCloseModal = () => setOpenModal(false); + + const handleChange = async (e) => { + setQuery(e.target.value); + } + + const handleSubmit = (e) => { + ws.close(); + if (query) { + openWebSocket(`(${query}) and leftOff(-1)`, true); } else { - setMethodsFilter([...methodsFilter, val]); + openWebSocket(`leftOff(-1)`, true); } + e.preventDefault(); } - return - methodClicked(val)} - transformDisplay={_toUpperCase} - label={"Methods"} - /> - ; -}; + return <> +
+ + + + + + + + + +
-export enum StatusType { - SUCCESS = "success", - ERROR = "error" + + + + + Filtering Guide (Cheatsheet) + + +

Mizu has a rich filtering syntax that let's you query the results both flexibly and efficiently.

+

Here are some examples that you can try;

+
+ + + + This is a simple query that matches to HTTP packets with request path "/catalogue": + + + + The same query can be negated for HTTP path and written like this: + + + + The syntax supports regular expressions. Here is a query that matches the HTTP requests that send JSON to a server: + + + + Here is another query that matches HTTP responses with status code 4xx: + + + + The same exact query can be as integer comparison: + + = 400`} + language="python" + /> + + The results can be queried based on their timestamps: + + + + + + + Since Mizu supports various protocols like gRPC, AMQP, Kafka and Redis. It's possible to write complex queries that match multiple protocols like this: + + + + By clicking the plus icon that appears beside the queryable UI elements on hovering in both left-pane and right-pane, you can automatically select a field and update the query: + + Clicking to UI elements (left-pane) + + Such that; clicking this icon in left-pane, would append the query below: + + + + Another queriable UI element example, this time from the right-pane: + + Clicking to UI elements (right-pane) + + A query that compares one selector to another is also a valid query: + + + + + + + There are a few helper methods included the in the filter language* to help building queries more easily. + +

+ + true if the given selector's value starts with the string: + + + + true if the given selector's value ends with the string: + + + + true if the given selector's value contains the string: + + + + returns the UNIX timestamp which is the equivalent of the time that's provided by the string. Invalid input evaluates to false: + + = datetime("10/19/2021, 6:29:02.593 PM")`} + language="python" + /> + + limits the number of records that are streamed back as a result of a query. Always evaluates to true: + + +
+
+

+ + *The filtering functionality is provided through Basenine database server. Please refer to BFL Syntax Reference for more information. + +
+
+
+ } - -interface StatusTypesFilterProps { - statusFilter: Array; - setStatusFilter: (methods: Array) => void; -} - -const StatusTypesFilter: React.FC = ({statusFilter, setStatusFilter}) => { - - const statusClicked = (val) => { - if(val === ALL_KEY) { - setStatusFilter([]); - return; - } - setStatusFilter([val]); - } - - return - statusClicked(val)} - transformDisplay={_toUpperCase} - label="Status" - /> - ; -}; - -interface PathFilterProps { - pathFilter: string; - setPathFilter: (val: string) => void; -} - -const PathFilter: React.FC = ({pathFilter, setPathFilter}) => { - - return -
Path
-
- setPathFilter(e.target.value)}/> -
-
; -}; - -interface ServiceFilterProps { - serviceFilter: string; - setServiceFilter: (val: string) => void; -} - -const ServiceFilter: React.FC = ({serviceFilter, setServiceFilter}) => { - - return -
Service
-
- setServiceFilter(e.target.value)}/> -
-
; -}; - diff --git a/ui/src/components/TrafficPage.tsx b/ui/src/components/TrafficPage.tsx index f1e3be2b1..0faf7a816 100644 --- a/ui/src/components/TrafficPage.tsx +++ b/ui/src/components/TrafficPage.tsx @@ -1,4 +1,4 @@ -import React, {useEffect, useRef, useState} from "react"; +import React, {useEffect, useMemo, useRef, useState} from "react"; import {Filters} from "./Filters"; import {EntriesList} from "./EntriesList"; import {makeStyles} from "@material-ui/core"; @@ -10,6 +10,9 @@ import pauseIcon from './assets/pause.svg'; import variables from '../variables.module.scss'; import {StatusBar} from "./UI/StatusBar"; import Api, {MizuWebsocketURL} from "../helpers/api"; +import { ToastContainer, toast } from 'react-toastify'; +import 'react-toastify/dist/ReactToastify.css'; +import debounce from 'lodash/debounce'; const useLayoutStyles = makeStyles(() => ({ details: { @@ -18,7 +21,7 @@ const useLayoutStyles = makeStyles(() => ({ padding: "12px 24px", borderRadius: 4, marginTop: 15, - background: variables.headerBackgoundColor, + background: variables.headerBackgroundColor, }, viewer: { @@ -34,7 +37,6 @@ const useLayoutStyles = makeStyles(() => ({ enum ConnectionStatus { Closed, Connected, - Paused } interface TrafficPageProps { @@ -52,26 +54,82 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS const [focusedEntryId, setFocusedEntryId] = useState(null); const [selectedEntryData, setSelectedEntryData] = useState(null); const [connection, setConnection] = useState(ConnectionStatus.Closed); - const [noMoreDataTop, setNoMoreDataTop] = useState(false); - const [noMoreDataBottom, setNoMoreDataBottom] = useState(false); - const [methodsFilter, setMethodsFilter] = useState([]); - const [statusFilter, setStatusFilter] = useState([]); - const [pathFilter, setPathFilter] = useState(""); - const [serviceFilter, setServiceFilter] = useState(""); + const [noMoreDataTop, setNoMoreDataTop] = useState(false); const [tappingStatus, setTappingStatus] = useState(null); - const [disableScrollList, setDisableScrollList] = useState(false); + const [isSnappedToBottom, setIsSnappedToBottom] = useState(true); + + const [query, setQuery] = useState(""); + const [queryBackgroundColor, setQueryBackgroundColor] = useState("#f5f5f5"); + const [addition, updateQuery] = useState(""); + + const [queriedCurrent, setQueriedCurrent] = useState(0); + const [queriedTotal, setQueriedTotal] = useState(0); + const [leftOffBottom, setLeftOffBottom] = useState(0); + const [leftOffTop, setLeftOffTop] = useState(null); + const [truncatedTimestamp, setTruncatedTimestamp] = useState(0); + + const [startTime, setStartTime] = useState(0); + + const handleQueryChange = useMemo(() => debounce(async (query: string) => { + if (!query) { + setQueryBackgroundColor("#f5f5f5") + } else { + const data = await api.validateQuery(query); + if (!data) { + return; + } + if (data.valid) { + setQueryBackgroundColor("#d2fad2"); + } else { + setQueryBackgroundColor("#fad6dc"); + } + } + }, 500), []) as (query: string) => void; + + useEffect(() => { + handleQueryChange(query); + }, [query]); + + useEffect(() => { + if (query) { + setQuery(`${query} and ${addition}`); + } else { + setQuery(addition); + } + // eslint-disable-next-line + }, [addition]); const ws = useRef(null); const listEntry = useRef(null); - const openWebSocket = () => { + const openWebSocket = (query: string, resetEntries: boolean) => { + if (resetEntries) { + setFocusedEntryId(null); + setEntries([]); + setQueriedCurrent(0); + setLeftOffTop(null); + setNoMoreDataTop(false); + } ws.current = new WebSocket(MizuWebsocketURL); - ws.current.onopen = () => setConnection(ConnectionStatus.Connected); - ws.current.onclose = () => setConnection(ConnectionStatus.Closed); + ws.current.onopen = () => { + setConnection(ConnectionStatus.Connected); + ws.current.send(query); + } + ws.current.onclose = () => { + setConnection(ConnectionStatus.Closed); + } + ws.current.onerror = (event) => { + console.error("WebSocket error:", event); + if (query) { + openWebSocket(`(${query}) and leftOff(${leftOffBottom})`, false); + } else { + openWebSocket(`leftOff(${leftOffBottom})`, false); + } + } } if (ws.current) { @@ -80,19 +138,15 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS const message = JSON.parse(e.data); switch (message.messageType) { case "entry": - const entry = message.data - if (connection === ConnectionStatus.Paused) { - setNoMoreDataBottom(false) - return; - } - if (!focusedEntryId) setFocusedEntryId(entry.id) - let newEntries = [...entries]; - setEntries([...newEntries, entry]) - if(listEntry.current) { - if(isScrollable(listEntry.current.firstChild)) { - setDisableScrollList(true) - } + const entry = message.data; + if (!focusedEntryId) setFocusedEntryId(entry.id.toString()) + const newEntries = [...entries, entry]; + if (newEntries.length === 10001) { + setLeftOffTop(newEntries[0].entry.id); + newEntries.shift(); + setNoMoreDataTop(false); } + setEntries(newEntries); break case "status": setTappingStatus(message.tappingStatus); @@ -103,6 +157,30 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS case "outboundLink": onTLSDetected(message.Data.DstIP); break; + case "toast": + toast[message.data.type](message.data.text, { + position: "bottom-right", + theme: "colored", + autoClose: message.data.autoClose, + hideProgressBar: false, + closeOnClick: true, + pauseOnHover: true, + draggable: true, + progress: undefined, + }); + break; + case "queryMetadata": + setQueriedCurrent(queriedCurrent + message.data.current); + setQueriedTotal(message.data.total); + setLeftOffBottom(message.data.leftOff); + setTruncatedTimestamp(message.data.truncatedTimestamp); + if (leftOffTop === null) { + setLeftOffTop(message.data.leftOff - 1); + } + break; + case "startTime": + setStartTime(message.data); + break; default: console.error(`unsupported websocket message type, Got: ${message.messageType}`) } @@ -111,7 +189,7 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS useEffect(() => { (async () => { - openWebSocket(); + openWebSocket("leftOff(-1)", true); try{ const tapStatusResponse = await api.tapStatus(); setTappingStatus(tapStatusResponse); @@ -133,20 +211,38 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS const entryData = await api.getEntry(focusedEntryId); setSelectedEntryData(entryData); } catch (error) { + if (error.response) { + toast[error.response.data.type](`Entry[${focusedEntryId}]: ${error.response.data.msg}`, { + position: "bottom-right", + theme: "colored", + autoClose: error.response.data.autoClose, + hideProgressBar: false, + closeOnClick: true, + pauseOnHover: true, + draggable: true, + progress: undefined, + }); + } console.error(error); } - })() - }, [focusedEntryId]) + })(); + // eslint-disable-next-line + }, [focusedEntryId]); const toggleConnection = () => { - setConnection(connection === ConnectionStatus.Connected ? ConnectionStatus.Paused : ConnectionStatus.Connected); + ws.current.close(); + if (connection !== ConnectionStatus.Connected) { + if (query) { + openWebSocket(`(${query}) and leftOff(-1)`, true); + } else { + openWebSocket(`leftOff(-1)`, true); + } + } } const getConnectionStatusClass = (isContainer) => { const container = isContainer ? "Container" : ""; switch (connection) { - case ConnectionStatus.Paused: - return "orangeIndicator" + container; case ConnectionStatus.Connected: return "greenIndicator" + container; default: @@ -156,28 +252,27 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS const getConnectionTitle = () => { switch (connection) { - case ConnectionStatus.Paused: - return "traffic paused"; case ConnectionStatus.Connected: - return "connected, waiting for traffic" + return "streaming live traffic" default: - return "not connected"; + return "streaming paused"; } } - const onScrollEvent = (isAtBottom) => { - isAtBottom ? setDisableScrollList(false) : setDisableScrollList(true) + const onSnapBrokenEvent = () => { + setIsSnappedToBottom(false); + if (connection === ConnectionStatus.Connected) { + ws.current.close(); + } } - const isScrollable = (element) => { - return element.scrollHeight > element.clientHeight; - }; - return (
- {connection !== ConnectionStatus.Closed && pause} + pause + play
{getConnectionTitle()}
@@ -185,42 +280,61 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS
- {entries.length > 0 &&
+ {
-
-
- {selectedEntryData && } + {selectedEntryData && }
} - {tappingStatus?.pods != null && } + {tappingStatus && } +
) }; diff --git a/ui/src/components/UI/EndpointPath.tsx b/ui/src/components/UI/EndpointPath.tsx deleted file mode 100644 index 2561aab44..000000000 --- a/ui/src/components/UI/EndpointPath.tsx +++ /dev/null @@ -1,15 +0,0 @@ -import miscStyles from "./style/misc.module.sass"; -import React from "react"; -import styles from './style/EndpointPath.module.sass'; - -interface EndpointPathProps { - method: string, - path: string -} - -export const EndpointPath: React.FC = ({method, path}) => { - return
- {method && {method}} - {path &&
{path}
} -
-}; diff --git a/ui/src/components/UI/FancyTextDisplay.tsx b/ui/src/components/UI/FancyTextDisplay.tsx index 91f10f4bf..10dc49f1c 100644 --- a/ui/src/components/UI/FancyTextDisplay.tsx +++ b/ui/src/components/UI/FancyTextDisplay.tsx @@ -17,7 +17,7 @@ interface Props { const FancyTextDisplay: React.FC = ({text, className, isPossibleToCopy = true, applyTextEllipsis = true, flipped = false, useTooltip= false, displayIconOnMouseOver = false, buttonOnly = false}) => { const [showCopiedNotification, setCopied] = useState(false); const [showTooltip, setShowTooltip] = useState(false); - const displayText = text || ''; + text = String(text); const onCopy = () => { setCopied(true) @@ -33,12 +33,12 @@ const FancyTextDisplay: React.FC = ({text, className, isPossibleToCopy = return () => clearTimeout(timer); }, [showCopiedNotification]); - const textElement = {displayText}; + const textElement = {text}; - const copyButton = isPossibleToCopy && displayText ? + const copyButton = isPossibleToCopy && text ? Duplicate full value {showCopiedNotification && Copied} @@ -48,14 +48,14 @@ const FancyTextDisplay: React.FC = ({text, className, isPossibleToCopy = return (

setShowTooltip(true)} onMouseLeave={ e => setShowTooltip(false)} > {!buttonOnly && flipped && textElement} {copyButton} {!buttonOnly && !flipped && textElement} - {useTooltip && showTooltip && {displayText}} + {useTooltip && showTooltip && {text}}

); }; diff --git a/ui/src/components/UI/FilterSelect.tsx b/ui/src/components/UI/FilterSelect.tsx deleted file mode 100644 index bf6764ad0..000000000 --- a/ui/src/components/UI/FilterSelect.tsx +++ /dev/null @@ -1,28 +0,0 @@ -import React from "react"; -import { MenuItem } from '@material-ui/core'; -import style from './style/FilterSelect.module.sass'; -import { Select, SelectProps } from "./Select"; - -interface FilterSelectProps extends SelectProps { - items: string[]; - value: string | string[]; - onChange: (string) => void; - label?: string; - allowMultiple?: boolean; - transformDisplay?: (string) => string; -} - -export const FilterSelect: React.FC = ({items, value, onChange, label, allowMultiple= false, transformDisplay}) => { - return -}; diff --git a/ui/src/components/UI/Protocol.tsx b/ui/src/components/UI/Protocol.tsx index 0b031bf18..3db0c2eb1 100644 --- a/ui/src/components/UI/Protocol.tsx +++ b/ui/src/components/UI/Protocol.tsx @@ -1,10 +1,12 @@ import React from "react"; import styles from './style/Protocol.module.sass'; +import Queryable from "./Queryable"; export interface ProtocolInterface { name: string longName: string - abbreviation: string + abbr: string + macro: string backgroundColor: string foregroundColor: string fontSize: number @@ -16,37 +18,51 @@ export interface ProtocolInterface { interface ProtocolProps { protocol: ProtocolInterface horizontal: boolean + updateQuery: any } -const Protocol: React.FC = ({protocol, horizontal}) => { +const Protocol: React.FC = ({protocol, horizontal, updateQuery}) => { if (horizontal) { - return - - {protocol.longName} - - + return + + + {protocol.longName} + + + } else { - return + return - {protocol.abbreviation} + {protocol.abbr} - + } }; diff --git a/ui/src/components/UI/Queryable.tsx b/ui/src/components/UI/Queryable.tsx new file mode 100644 index 000000000..e08d36cd9 --- /dev/null +++ b/ui/src/components/UI/Queryable.tsx @@ -0,0 +1,62 @@ +import React, { useEffect, useState } from 'react'; +import { CopyToClipboard } from 'react-copy-to-clipboard'; +import AddCircleIcon from '@material-ui/icons/AddCircle'; +import './style/Queryable.sass'; + +interface Props { + query: string, + updateQuery: any, + style?: object, + iconStyle?: object, + className?: string, + useTooltip?: boolean, + displayIconOnMouseOver?: boolean, + flipped?: boolean, +} + +const Queryable: React.FC = ({query, updateQuery, style, iconStyle, className, useTooltip= true, displayIconOnMouseOver = false, flipped = false, children}) => { + const [showAddedNotification, setAdded] = useState(false); + const [showTooltip, setShowTooltip] = useState(false); + + const onCopy = () => { + setAdded(true) + }; + + useEffect(() => { + let timer; + if (showAddedNotification) { + updateQuery(query); + timer = setTimeout(() => { + setAdded(false); + }, 1000); + } + return () => clearTimeout(timer); + }, [showAddedNotification, query, updateQuery]); + + const addButton = query ? + + + {showAddedNotification && Added} + + : null; + + return ( +
setShowTooltip(true)} + onMouseLeave={ e => setShowTooltip(false)} + > + {flipped && addButton} + {children} + {!flipped && addButton} + {useTooltip && showTooltip && {query}} +
+ ); +}; + +export default Queryable; diff --git a/ui/src/components/UI/StatusBar.tsx b/ui/src/components/UI/StatusBar.tsx index 5fe6d2738..d8f71b48b 100644 --- a/ui/src/components/UI/StatusBar.tsx +++ b/ui/src/components/UI/StatusBar.tsx @@ -1,9 +1,13 @@ import './style/StatusBar.sass'; import React, {useState} from "react"; +import warningIcon from '../assets/warning_icon.svg'; +import failIcon from '../assets/failed.svg'; +import successIcon from '../assets/success.svg'; export interface TappingStatusPod { name: string; namespace: string; + isTapped: boolean; } export interface TappingStatus { @@ -11,7 +15,7 @@ export interface TappingStatus { } export interface Props { - tappingStatus: TappingStatus + tappingStatus: TappingStatusPod[] } const pluralize = (noun: string, amount: number) => { @@ -22,23 +26,29 @@ export const StatusBar: React.FC = ({tappingStatus}) => { const [expandedBar, setExpandedBar] = useState(false); - const uniqueNamespaces = Array.from(new Set(tappingStatus.pods.map(pod => pod.namespace))); - const amountOfPods = tappingStatus.pods.length; + const uniqueNamespaces = Array.from(new Set(tappingStatus.map(pod => pod.namespace))); + const amountOfPods = tappingStatus.length; + const amountOfTappedPods = tappingStatus.filter(pod => pod.isTapped).length; + const amountOfUntappedPods = amountOfPods - amountOfTappedPods; return
setExpandedBar(true)} onMouseLeave={() => setExpandedBar(false)}> -
{`Tapping ${amountOfPods} ${pluralize('pod', amountOfPods)} in ${pluralize('namespace', uniqueNamespaces.length)} ${uniqueNamespaces.join(", ")}`}
+
+ {tappingStatus.some(pod => !pod.isTapped) && warning} + {`Tapping ${amountOfUntappedPods > 0 ? amountOfTappedPods + " / " + amountOfPods : amountOfPods} ${pluralize('pod', amountOfPods)} in ${pluralize('namespace', uniqueNamespaces.length)} ${uniqueNamespaces.join(", ")}`}
{expandedBar &&
+ - {tappingStatus.pods.map(pod => + {tappingStatus.map(pod => + )}
Pod name NamespaceTapping
{pod.name} {pod.namespace}status
diff --git a/ui/src/components/UI/StatusCode.tsx b/ui/src/components/UI/StatusCode.tsx index b35788ac1..054be378a 100644 --- a/ui/src/components/UI/StatusCode.tsx +++ b/ui/src/components/UI/StatusCode.tsx @@ -1,5 +1,6 @@ import React from "react"; import styles from './style/StatusCode.module.sass'; +import Queryable from "./Queryable"; export enum StatusCodeClassification { SUCCESS = "success", @@ -9,17 +10,27 @@ export enum StatusCodeClassification { interface EntryProps { statusCode: number + updateQuery: any } -const StatusCode: React.FC = ({statusCode}) => { +const StatusCode: React.FC = ({statusCode, updateQuery}) => { const classification = getClassification(statusCode) - return + return + {statusCode} - + + }; export function getClassification(statusCode: number): string { diff --git a/ui/src/components/UI/Summary.tsx b/ui/src/components/UI/Summary.tsx new file mode 100644 index 000000000..760f0f5f3 --- /dev/null +++ b/ui/src/components/UI/Summary.tsx @@ -0,0 +1,38 @@ +import miscStyles from "./style/misc.module.sass"; +import React from "react"; +import styles from './style/Summary.module.sass'; +import Queryable from "./Queryable"; + +interface SummaryProps { + method: string + summary: string + updateQuery: any +} + +export const Summary: React.FC = ({method, summary, updateQuery}) => { + + return
+ {method && + + {method} + + } + {summary && +
+ {summary} +
+
} +
+}; diff --git a/ui/src/components/UI/SyntaxHighlighter/highlighterStyle.ts b/ui/src/components/UI/SyntaxHighlighter/highlighterStyle.ts index a5be67b25..1766e309a 100644 --- a/ui/src/components/UI/SyntaxHighlighter/highlighterStyle.ts +++ b/ui/src/components/UI/SyntaxHighlighter/highlighterStyle.ts @@ -112,7 +112,7 @@ export const highlighterStyle = { "color": "#C6C5FE" }, "operator": { - "color": "#EDEDED" + "color": "#A1A1A1" }, "entity": { "color": "#fdab2b", diff --git a/ui/src/components/UI/style/EndpointPath.module.sass b/ui/src/components/UI/style/EndpointPath.module.sass deleted file mode 100644 index 2fc54c0f3..000000000 --- a/ui/src/components/UI/style/EndpointPath.module.sass +++ /dev/null @@ -1,8 +0,0 @@ -.container - display: flex - align-items: center - -.path - text-overflow: ellipsis - overflow: hidden - white-space: nowrap \ No newline at end of file diff --git a/ui/src/components/UI/style/Protocol.module.sass b/ui/src/components/UI/style/Protocol.module.sass index e702f5ae2..9ffeaab14 100644 --- a/ui/src/components/UI/style/Protocol.module.sass +++ b/ui/src/components/UI/style/Protocol.module.sass @@ -6,7 +6,6 @@ background-color: #000 color: #fff margin-left: -8px - margin-bottom: -4px .vertical line-height: 22px diff --git a/ui/src/components/UI/style/Queryable.sass b/ui/src/components/UI/style/Queryable.sass new file mode 100644 index 000000000..f045f2183 --- /dev/null +++ b/ui/src/components/UI/style/Queryable.sass @@ -0,0 +1,48 @@ +.Queryable-Container + display: flex + align-items: center + + &.displayIconOnMouseOver + .Queryable-Icon + opacity: 0 + width: 0px + pointer-events: none + &:hover + .Queryable-Icon + opacity: 1 + pointer-events: all + + + .Queryable-Icon + height: 22px + width: 22px + cursor: pointer + color: #27AE60 + + &:hover + background-color: rgba(255, 255, 255, 0.06) + border-radius: 4px + color: #1E884B + + .Queryable-AddNotifier + background-color: #1E884B + font-weight: normal + padding: 2px 5px + border-radius: 4px + position: absolute + transform: translate(0, 10%) + color: white + z-index: 1000 + font-size: 11px + + .Queryable-Tooltip + background-color: #1E884B + font-weight: normal + padding: 2px 5px + border-radius: 4px + position: absolute + transform: translate(0, -80%) + color: white + z-index: 1000 + font-size: 11px + diff --git a/ui/src/components/UI/style/StatusBar.sass b/ui/src/components/UI/style/StatusBar.sass index 6a55dbc36..f6e66624c 100644 --- a/ui/src/components/UI/style/StatusBar.sass +++ b/ui/src/components/UI/style/StatusBar.sass @@ -24,8 +24,13 @@ padding: 8px font-weight: 600 + img + margin-right: 10px + height: 22px + th text-align: left + padding-right: 15px td padding-right: 15px padding-top: 5px diff --git a/ui/src/components/UI/style/Summary.module.sass b/ui/src/components/UI/style/Summary.module.sass new file mode 100644 index 000000000..2bce9af67 --- /dev/null +++ b/ui/src/components/UI/style/Summary.module.sass @@ -0,0 +1,6 @@ +.container + display: flex + align-items: center + +.summary + white-space: nowrap diff --git a/ui/src/components/UI/style/misc.module.sass b/ui/src/components/UI/style/misc.module.sass index 8f57e3382..bef317cd3 100644 --- a/ui/src/components/UI/style/misc.module.sass +++ b/ui/src/components/UI/style/misc.module.sass @@ -5,20 +5,20 @@ border: solid 1px $secondary-font-color margin-left: 4px padding: 2px 5px - text-transform: uppercase font-family: "Source Sans Pro", sans-serif font-size: 11px font-weight: bold &.method margin-right: 10px + height: 12px &.filterPlate border-color: #bcc6dd20 color: #a0b2ff font-size: 10px -.noSelect +.noSelect -webkit-touch-callout: none -webkit-user-select: none -khtml-user-select: none diff --git a/ui/src/components/assets/failed.svg b/ui/src/components/assets/failed.svg new file mode 100644 index 000000000..bab53af12 --- /dev/null +++ b/ui/src/components/assets/failed.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/ui/src/components/assets/filter-ui-example-1.png b/ui/src/components/assets/filter-ui-example-1.png new file mode 100644 index 000000000..6f4d88430 Binary files /dev/null and b/ui/src/components/assets/filter-ui-example-1.png differ diff --git a/ui/src/components/assets/filter-ui-example-2.png b/ui/src/components/assets/filter-ui-example-2.png new file mode 100644 index 000000000..e8ae1a5ec Binary files /dev/null and b/ui/src/components/assets/filter-ui-example-2.png differ diff --git a/ui/src/components/assets/success.svg b/ui/src/components/assets/success.svg new file mode 100644 index 000000000..f8fe3aa64 --- /dev/null +++ b/ui/src/components/assets/success.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/ui/src/components/assets/warning_icon.svg b/ui/src/components/assets/warning_icon.svg new file mode 100644 index 000000000..7ef6ba5a2 --- /dev/null +++ b/ui/src/components/assets/warning_icon.svg @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ui/src/components/style/EntriesList.module.sass b/ui/src/components/style/EntriesList.module.sass index 453f46081..5295dd656 100644 --- a/ui/src/components/style/EntriesList.module.sass +++ b/ui/src/components/style/EntriesList.module.sass @@ -47,13 +47,21 @@ text-align: center font-weight: 600 color: $secondary-font-color -.fetchButtonContainer - width: 100% - display: flex - justify-content: center - margin-top: 12px - font-weight: 600 - color: rgba(255,255,255,0.75) + +.btnOld + position: absolute + top: 20px + right: 10px + background: #205CF5 + border-radius: 50% + height: 35px + width: 35px + border: none + cursor: pointer + z-index: 1 + img + height: 10px + transform: scaleY(-1) .btnLive position: absolute diff --git a/ui/src/components/style/Filters.module.sass b/ui/src/components/style/Filters.module.sass index ae8af8224..a815d7a9d 100644 --- a/ui/src/components/style/Filters.module.sass +++ b/ui/src/components/style/Filters.module.sass @@ -4,9 +4,6 @@ display: flex flex-direction: row align-items: center - min-height: 3rem - overflow-y: hidden - overflow-x: auto padding: .5rem 0 border-bottom: 1px solid #BCC6DD margin-right: 20px @@ -29,8 +26,24 @@ input padding: 4px 12px background: $main-background-color - border-radius: 12px - font-size: 12px + border-radius: 4px + font-size: 14px border: 1px solid #BCC6DD fieldset border: none + +$divider-breakpoint-1: 1474px +$divider-breakpoint-2: 1366px +$divider-breakpoint-3: 1980px + +@media (max-width: $divider-breakpoint-1) + .divider1 + display: none + +@media (max-width: $divider-breakpoint-2) + .divider2 + display: none + +@media (min-width: $divider-breakpoint-1) and (max-width: $divider-breakpoint-3) + .divider2 + display: none diff --git a/ui/src/components/style/TrafficPage.sass b/ui/src/components/style/TrafficPage.sass index ceeefe128..68d4c3d08 100644 --- a/ui/src/components/style/TrafficPage.sass +++ b/ui/src/components/style/TrafficPage.sass @@ -110,3 +110,8 @@ align-items: center height: 17px font-size: 16px + +.playPauseIcon + cursor: pointer + margin-right: 15px + height: 30px diff --git a/ui/src/helpers/api.js b/ui/src/helpers/api.js index 216b5b0c4..ef7404996 100644 --- a/ui/src/helpers/api.js +++ b/ui/src/helpers/api.js @@ -1,16 +1,16 @@ import * as axios from "axios"; -const mizuAPIPathPrefix = "/mizu"; - // When working locally cp `cp .env.example .env` -export const MizuWebsocketURL = process.env.REACT_APP_OVERRIDE_WS_URL ? process.env.REACT_APP_OVERRIDE_WS_URL : `ws://${window.location.host}${mizuAPIPathPrefix}/ws`; +export const MizuWebsocketURL = process.env.REACT_APP_OVERRIDE_WS_URL ? process.env.REACT_APP_OVERRIDE_WS_URL : `ws://${window.location.host}/ws`; + +const CancelToken = axios.CancelToken; export default class Api { constructor() { // When working locally cp `cp .env.example .env` - const apiURL = process.env.REACT_APP_OVERRIDE_API_URL ? process.env.REACT_APP_OVERRIDE_API_URL : `${window.location.origin}${mizuAPIPathPrefix}/`; + const apiURL = process.env.REACT_APP_OVERRIDE_API_URL ? process.env.REACT_APP_OVERRIDE_API_URL : `${window.location.origin}/`; this.client = axios.create({ baseURL: apiURL, @@ -19,6 +19,8 @@ export default class Api { Accept: "application/json", } }); + + this.source = null; } tapStatus = async () => { @@ -31,13 +33,16 @@ export default class Api { return response.data; } - getEntry = async (entryId) => { - const response = await this.client.get(`/entries/${entryId}`); + getEntry = async (id) => { + const response = await this.client.get(`/entries/${id}`); return response.data; } - fetchEntries = async (operator, timestamp) => { - const response = await this.client.get(`/entries?limit=50&operator=${operator}×tamp=${timestamp}`); + fetchEntries = async (leftOff, direction, query, limit, timeoutMs) => { + const response = await this.client.get(`/entries/?leftOff=${leftOff}&direction=${direction}&query=${query}&limit=${limit}&timeoutMs=${timeoutMs}`).catch(function (thrown) { + console.error(thrown.message); + return {}; + }); return response.data; } @@ -50,4 +55,27 @@ export default class Api { const response = await this.client.get("/status/auth"); return response.data; } + + validateQuery = async (query) => { + if (this.source) { + this.source.cancel(); + } + this.source = CancelToken.source(); + + const form = new FormData(); + form.append('query', query) + const response = await this.client.post(`/query/validate`, form, { + cancelToken: this.source.token + }).catch(function (thrown) { + if (!axios.isCancel(thrown)) { + console.error('Validate error', thrown.message); + } + }); + + if (!response) { + return null; + } + + return response.data; + } } diff --git a/ui/src/variables.module.scss b/ui/src/variables.module.scss index d2f3d89bd..9e25bb3c3 100644 --- a/ui/src/variables.module.scss +++ b/ui/src/variables.module.scss @@ -11,7 +11,7 @@ $blue-gray: #494677; :export { mainBackgroundColor: $main-background-color; - headerBackgoundColor: $header-background-color; + headerBackgroundColor: $header-background-color; fontColor: $font-color; secondaryFontColor: $secondary-font-color; blueColor: $blue-color;