diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..694076f95 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,4 @@ +# This is a comment. +# Each line is a file pattern followed by one or more owners. + +/ui/ @frontend diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..ff5a7ecfd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Run mizu '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Logs** +Upload logs: +1. Run the mizu command with `--set dump-logs=true` (e.g `mizu tap --set dump-logs=true`) +2. Try to reproduce the issue +3. CNTRL+C on terminal tab which runs mizu +4. Upload the logs zip file from ~/.mizu/mizu_logs_**.zip + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Browser [e.g. chrome] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/workflows/acceptance_tests.yml b/.github/workflows/acceptance_tests.yml new file mode 100644 index 000000000..a23f3c445 --- /dev/null +++ b/.github/workflows/acceptance_tests.yml @@ -0,0 +1,32 @@ +name: acceptance tests + +on: + pull_request: + branches: + - 'main' + push: + branches: + - 'develop' + +concurrency: + group: mizu-acceptance-tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + run-acceptance-tests: + name: Run acceptance tests + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.16 + uses: actions/setup-go@v2 + with: + go-version: '^1.16' + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Setup acceptance test + run: source ./acceptanceTests/setup.sh + + - name: Test + run: make acceptance-test diff --git a/.github/workflows/pr_validation.yml b/.github/workflows/pr_validation.yml new file mode 100644 index 000000000..08b8388b5 --- /dev/null +++ b/.github/workflows/pr_validation.yml @@ -0,0 +1,80 @@ +name: PR validation +on: + pull_request: + branches: + - 'develop' + - 'main' +jobs: + build-cli: + name: Build CLI + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.16 + uses: actions/setup-go@v2 + with: + go-version: '^1.16' + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Build CLI + run: make cli + + build-agent: + name: Build Agent + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.16 + uses: actions/setup-go@v2 + with: + go-version: '^1.16' + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - shell: bash + run: | + sudo apt-get install libpcap-dev + + - name: Build Agent + run: make agent + + run-tests-cli: + name: Run CLI tests + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.16 + uses: actions/setup-go@v2 + with: + go-version: '^1.16' + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Test + run: make test-cli + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + + run-tests-agent: + name: Run Agent tests + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.16 + uses: actions/setup-go@v2 + with: + go-version: '^1.16' + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - shell: bash + run: | + sudo apt-get install libpcap-dev + + - name: Test + run: make test-agent + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 531e75f34..e0fb1523a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,9 +1,15 @@ name: publish + on: push: branches: - 'develop' - 'main' + +concurrency: + group: mizu-publish-${{ github.ref }} + cancel-in-progress: true + jobs: docker: runs-on: ubuntu-latest @@ -78,4 +84,3 @@ jobs: tag: ${{ steps.versioning.outputs.version }} prerelease: ${{ github.ref != 'refs/heads/main' }} bodyFile: 'cli/bin/README.md' - diff --git a/CONTRIBUTE.md b/CONTRIBUTE.md new file mode 100644 index 000000000..21652cca5 --- /dev/null +++ b/CONTRIBUTE.md @@ -0,0 +1,18 @@ +![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg) +# CONTRIBUTE +We welcome code contributions from the community. +Please read and follow the guidelines below. + +## Communication +* Before starting work on a major feature, please reach out to us via [GitHub](https://github.com/up9inc/mizu), [Slack](https://join.slack.com/share/zt-u6bbs3pg-X1zhQOXOH0yEoqILgH~csw), [email](mailto:mizu@up9.com), etc. We will make sure no one else is already working on it. A _major feature_ is defined as any change that is > 100 LOC altered (not including tests), or changes any user-facing behavior +* Small patches and bug fixes don't need prior communication. + +## Contribution requirements +* Code style - most of the code is written in Go, please follow [these guidelines](https://golang.org/doc/effective_go) +* Go-tools compatible (`go get`, `go test`, etc) +* Unit-test coverage can’t go down .. +* Code must be usefully commented. Not only for developers on the project, but also for external users of these packages +* When reviewing PRs, you are encouraged to use Golang's [code review comments page](https://github.com/golang/go/wiki/CodeReviewComments) + + + diff --git a/Dockerfile b/Dockerfile index c19b1805f..a3864f126 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,8 +48,6 @@ WORKDIR /app COPY --from=builder ["/app/agent-build/mizuagent", "."] COPY --from=site-build ["/app/ui-build/build", "site"] -COPY agent/start.sh . - # gin-gonic runs in debug mode without this ENV GIN_MODE=release diff --git a/Makefile b/Makefile index bb85ff8ea..2807c6283 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,9 @@ ui: ## Build UI. cli: ## Build CLI. @echo "building cli"; cd cli && $(MAKE) build +build-cli-ci: ## Build CLI for CI. + @echo "building cli for ci"; cd cli && $(MAKE) build GIT_BRANCH=ci SUFFIX=ci + agent: ## Build agent. @(echo "building mizu agent .." ) @(cd agent; go build -o build/mizuagent main.go) @@ -42,6 +45,10 @@ push-docker: ## Build and publish agent docker image. @echo "publishing Docker image .. " ./build-push-featurebranch.sh +build-docker-ci: ## Build agent docker image for CI. + @echo "building docker image for ci" + ./build-agent-ci.sh + push-cli: ## Build and publish CLI. @echo "publishing CLI .. " @cd cli; $(MAKE) build-all @@ -50,7 +57,6 @@ push-cli: ## Build and publish CLI. gsutil cp -r ./cli/bin/* gs://${BUCKET_PATH}/ gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\* - clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts. clean-ui: ## Clean UI. @@ -65,3 +71,11 @@ clean-cli: ## Clean CLI. clean-docker: @(echo "DOCKER cleanup - NOT IMPLEMENTED YET " ) +test-cli: + @echo "running cli tests"; cd cli && $(MAKE) test + +test-agent: + @echo "running agent tests"; cd agent && $(MAKE) test + +acceptance-test: + @echo "running acceptance tests"; cd acceptanceTests && $(MAKE) test diff --git a/README.md b/README.md index 91734b4c0..6993e029b 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,17 @@ ![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg) + # The API Traffic Viewer for Kubernetes -A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined. +A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined ![Simple UI](assets/mizu-ui.png) ## Features - Simple and powerful CLI -- Real time view of all HTTP requests, REST and gRPC API calls +- Real-time view of all HTTP requests, REST and gRPC API calls - No installation or code instrumentation -- Works completely on premises (on-prem) +- Works completely on premises ## Download @@ -32,10 +33,10 @@ https://github.com/up9inc/mizu/releases/latest/download/mizu_linux_amd64 \ && chmod 755 mizu ``` -SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page. +SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page ### Development (unstable) Build -Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page. +Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page ## Prerequisites 1. Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config` @@ -48,8 +49,8 @@ For detailed list of k8s permissions see [PERMISSIONS](PERMISSIONS.md) document 1. Find pods you'd like to tap to in your Kubernetes cluster 2. Run `mizu tap` or `mizu tap PODNAME` -3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI .. -4. Watch the API traffic flowing .. +3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI +4. Watch the API traffic flowing 5. Type ^C to stop ## Examples @@ -75,7 +76,7 @@ To tap all pods in current namespace - To tap specific pod - -``` +```bash $ kubectl get pods NAME READY STATUS RESTARTS AGE front-end-649fc5fd6-kqbtn 2/2 Running 0 7m @@ -88,7 +89,7 @@ To tap specific pod - ``` To tap multiple pods using regex - -``` +```bash $ kubectl get pods NAME READY STATUS RESTARTS AGE carts-66c77f5fbb-fq65r 2/2 Running 0 20m @@ -106,30 +107,64 @@ To tap multiple pods using regex - ## Configuration Mizu can work with config file which should be stored in ${HOME}/.mizu/config.yaml (macOS: ~/.mizu/config.yaml)
-In case no config file found, defaults will be used.
-In case of partial configuration defined, all other fields will be used with defaults.
-You can always override the defaults or config file with CLI flags. +In case no config file found, defaults will be used
+In case of partial configuration defined, all other fields will be used with defaults
+You can always override the defaults or config file with CLI flags To get the default config params run `mizu config`
To generate a new config file with default values use `mizu config -r` -Mizu has several undocumented flags which can be set by using --set flag (e.g., `mizu tap --set dump-logs=true`) -* **mizu-resources-namespace**: Type - String, See [Namespace-Restricted Mode](#namespace-restricted-mode) -* **telemetry**: Type - Boolean, Reports telemetry -* **dump-logs**: Type - Boolean, At the end of the execution it creates a zip file with logs (in .mizu folder) -* **kube-config-path**: Type - String, Setting the path to kube config (which isn't in standard path) +### Telemetry + +By default, mizu reports usage telemetry. It can be disabled by adding a line of `telemetry: false` in the `${HOME}/.mizu/config.yaml` file + ## Advanced Usage ### Namespace-Restricted Mode -Some users have permission to only manage resources in one particular namespace assigned to them. +Some users have permission to only manage resources in one particular namespace assigned to them By default `mizu tap` creates a new namespace `mizu` for all of its Kubernetes resources. In order to instead install -Mizu in an existing namespace, set the `mizu-resources-namespace` config option. +Mizu in an existing namespace, set the `mizu-resources-namespace` config option If `mizu-resources-namespace` is set to a value other than the default `mizu`, Mizu will operate in a Namespace-Restricted mode. It will only tap pods in `mizu-resources-namespace`. This way Mizu only requires permissions to the namespace set by `mizu-resources-namespace`. The user must set the tapped namespace to the same namespace by -using the `--namespace` flag or by setting `tap.namespaces` in the config file. +using the `--namespace` flag or by setting `tap.namespaces` in the config file -Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior. +Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior + +### User agent filtering + +User-agent filtering (like health checks) - can be configured using command-line options: + +```shell +$ mizu tap "^ca.*" --set ignored-user-agents=kube-probe --set ignored-user-agents=prometheus ++carts-66c77f5fbb-fq65r ++catalogue-5f4cb7cf5-7zrmn +Web interface is now available at http://localhost:8899 +^C + +``` + +Any request that contains `User-Agent` header with one of the specified values (`kube-probe` or `prometheus`) will not be captured + +### API Rules validation + +This feature allows you to define set of simple rules, and test the API against them. +Such validation may test response for specific JSON fields, headers, etc. + +Please see [API RULES](docs/POLICY_RULES.md) page for more details and syntax. + + +## How to Run local UI + +- run from mizu/agent `go run main.go --hars-read --hars-dir ` + +- copy Har files into the folder from last command + +- change `MizuWebsocketURL` and `apiURL` in `api.js` file + +- run from mizu/ui - `npm run start` + +- open browser on `localhost:3000` diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 000000000..a9e7b793b --- /dev/null +++ b/TESTING.md @@ -0,0 +1,15 @@ +![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg) +# TESTING +Testing guidelines for Mizu project + +## Unit-tests +* TBD +* TBD +* TBD + + + +## System tests +* TBD +* TBD +* TBD diff --git a/acceptanceTests/Makefile b/acceptanceTests/Makefile new file mode 100644 index 000000000..8a142a160 --- /dev/null +++ b/acceptanceTests/Makefile @@ -0,0 +1,2 @@ +test: ## Run acceptance tests. + @go test ./... diff --git a/acceptanceTests/go.mod b/acceptanceTests/go.mod new file mode 100644 index 000000000..bc529ac9d --- /dev/null +++ b/acceptanceTests/go.mod @@ -0,0 +1,3 @@ +module github.com/up9inc/mizu/tests + +go 1.16 diff --git a/acceptanceTests/setup.sh b/acceptanceTests/setup.sh new file mode 100644 index 000000000..0ed9dbce1 --- /dev/null +++ b/acceptanceTests/setup.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +PREFIX=$HOME/local/bin +VERSION=v1.22.0 + +echo "Attempting to install minikube and assorted tools to $PREFIX" + +if ! [ -x "$(command -v kubectl)" ]; then + echo "Installing kubectl version $VERSION" + curl -LO "https://storage.googleapis.com/kubernetes-release/release/$VERSION/bin/linux/amd64/kubectl" + chmod +x kubectl + mv kubectl "$PREFIX" +else + echo "kubetcl is already installed" +fi + +if ! [ -x "$(command -v minikube)" ]; then + echo "Installing minikube version $VERSION" + curl -Lo minikube https://storage.googleapis.com/minikube/releases/$VERSION/minikube-linux-amd64 + chmod +x minikube + mv minikube "$PREFIX" +else + echo "minikube is already installed" +fi + +echo "Starting minikube..." +minikube start + +echo "Creating mizu tests namespace" +kubectl create namespace mizu-tests + +echo "Creating httpbin deployment" +kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests + +echo "Creating httpbin service" +kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests + +echo "Starting proxy" +kubectl proxy --port=8080 & + +echo "Setting minikube docker env" +eval $(minikube docker-env) + +echo "Build agent image" +make build-docker-ci + +echo "Build cli" +make build-cli-ci diff --git a/acceptanceTests/tap_test.go b/acceptanceTests/tap_test.go new file mode 100644 index 000000000..3c359a0d0 --- /dev/null +++ b/acceptanceTests/tap_test.go @@ -0,0 +1,125 @@ +package acceptanceTests + +import ( + "fmt" + "io/ioutil" + "os/exec" + "testing" + "time" +) + +func TestTapAndFetch(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + tests := []int{1, 100} + + for _, entriesCount := range tests { + t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) { + cliPath, cliPathErr := GetCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + tapCmdArgs := GetDefaultTapCommandArgs() + tapCmd := exec.Command(cliPath, tapCmdArgs...) + t.Logf("running command: %v", tapCmd.String()) + + t.Cleanup(func() { + if err := CleanupCommand(tapCmd); err != nil { + t.Logf("failed to cleanup tap command, err: %v", err) + } + }) + + if err := tapCmd.Start(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + time.Sleep(30 * time.Second) + + proxyUrl := "http://localhost:8080/api/v1/namespaces/mizu-tests/services/httpbin/proxy/get" + for i := 0; i < entriesCount; i++ { + if _, requestErr := ExecuteHttpRequest(proxyUrl); requestErr != nil { + t.Errorf("failed to send proxy request, err: %v", requestErr) + return + } + } + + time.Sleep(5 * time.Second) + timestamp := time.Now().UnixNano() / int64(time.Millisecond) + + entriesUrl := fmt.Sprintf("http://localhost:8899/mizu/api/entries?limit=%v&operator=lt×tamp=%v", entriesCount, timestamp) + requestResult, requestErr := ExecuteHttpRequest(entriesUrl) + if requestErr != nil { + t.Errorf("failed to get entries, err: %v", requestErr) + return + } + + entries, ok := requestResult.([]interface{}) + if !ok { + t.Errorf("invalid entries type") + return + } + + if len(entries) != entriesCount { + t.Errorf("unexpected entries result - Expected: %v, actual: %v", entriesCount, len(entries)) + return + } + + entry, ok := entries[0].(map[string]interface{}) + if !ok { + t.Errorf("invalid entry type") + return + } + + entryUrl := fmt.Sprintf("http://localhost:8899/mizu/api/entries/%v", entry["id"]) + requestResult, requestErr = ExecuteHttpRequest(entryUrl) + if requestErr != nil { + t.Errorf("failed to get entry, err: %v", requestErr) + return + } + + if requestResult == nil { + t.Errorf("unexpected nil entry result") + return + } + + fetchCmdArgs := GetDefaultFetchCommandArgs() + fetchCmd := exec.Command(cliPath, fetchCmdArgs...) + t.Logf("running command: %v", fetchCmd.String()) + + t.Cleanup(func() { + if err := CleanupCommand(fetchCmd); err != nil { + t.Logf("failed to cleanup fetch command, err: %v", err) + } + }) + + if err := fetchCmd.Start(); err != nil { + t.Errorf("failed to start fetch command, err: %v", err) + return + } + + time.Sleep(5 * time.Second) + + harBytes, readFileErr := ioutil.ReadFile("./unknown_source.har") + if readFileErr != nil { + t.Errorf("failed to read har file, err: %v", readFileErr) + return + } + + harEntries, err := GetEntriesFromHarBytes(harBytes) + if err != nil { + t.Errorf("failed to get entries from har, err: %v", err) + return + } + + if len(harEntries) != entriesCount { + t.Errorf("unexpected har entries result - Expected: %v, actual: %v", entriesCount, len(harEntries)) + return + } + }) + } +} diff --git a/acceptanceTests/testsUtils.go b/acceptanceTests/testsUtils.go new file mode 100644 index 000000000..123151ac3 --- /dev/null +++ b/acceptanceTests/testsUtils.go @@ -0,0 +1,113 @@ +package acceptanceTests + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path" + "syscall" +) + +func GetCliPath() (string, error) { + dir, filePathErr := os.Getwd() + if filePathErr != nil { + return "", filePathErr + } + + cliPath := path.Join(dir, "../cli/bin/mizu_ci") + return cliPath, nil +} + +func GetDefaultCommandArgs() []string { + setFlag := "--set" + telemetry := "telemetry=false" + + return []string{setFlag, telemetry} +} + +func GetDefaultTapCommandArgs() []string { + tapCommand := "tap" + setFlag := "--set" + namespaces := "tap.namespaces=mizu-tests" + agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0" + imagePullPolicy := "image-pull-policy=Never" + + defaultCmdArgs := GetDefaultCommandArgs() + + return append([]string{tapCommand, setFlag, namespaces, setFlag, agentImage, setFlag, imagePullPolicy}, defaultCmdArgs...) +} + +func GetDefaultFetchCommandArgs() []string { + tapCommand := "fetch" + + defaultCmdArgs := GetDefaultCommandArgs() + + return append([]string{tapCommand}, defaultCmdArgs...) +} + +func JsonBytesToInterface(jsonBytes []byte) (interface{}, error) { + var result interface{} + if parseErr := json.Unmarshal(jsonBytes, &result); parseErr != nil { + return nil, parseErr + } + + return result, nil +} + +func ExecuteHttpRequest(url string) (interface{}, error) { + response, requestErr := http.Get(url) + if requestErr != nil { + return nil, requestErr + } else if response.StatusCode != 200 { + return nil, fmt.Errorf("invalid status code %v", response.StatusCode) + } + + data, readErr := ioutil.ReadAll(response.Body) + if readErr != nil { + return nil, readErr + } + + return JsonBytesToInterface(data) +} + +func CleanupCommand(cmd *exec.Cmd) error { + if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil { + return err + } + + if err := cmd.Wait(); err != nil { + return err + } + + return nil +} + +func GetEntriesFromHarBytes(harBytes []byte) ([]interface{}, error){ + harInterface, convertErr := JsonBytesToInterface(harBytes) + if convertErr != nil { + return nil, convertErr + } + + har, ok := harInterface.(map[string]interface{}) + if !ok { + return nil, errors.New("invalid har type") + } + + harLogInterface := har["log"] + harLog, ok := harLogInterface.(map[string]interface{}) + if !ok { + return nil, errors.New("invalid har log type") + } + + harEntriesInterface := harLog["entries"] + harEntries, ok := harEntriesInterface.([]interface{}) + if !ok { + return nil, errors.New("invalid har entries type") + } + + return harEntries, nil +} diff --git a/agent/Makefile b/agent/Makefile new file mode 100644 index 000000000..3ae43e684 --- /dev/null +++ b/agent/Makefile @@ -0,0 +1,2 @@ +test: ## Run agent tests. + @go test ./... -coverpkg=./... -race -coverprofile=coverage.out -covermode=atomic diff --git a/agent/go.sum b/agent/go.sum index 0917a0fbb..a0b6f00fa 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -540,8 +540,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM= gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw= gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= diff --git a/agent/main.go b/agent/main.go index 381433832..8f0ffd4d6 100644 --- a/agent/main.go +++ b/agent/main.go @@ -26,14 +26,17 @@ var apiServerMode = flag.Bool("api-server", false, "Run in API server mode with var standaloneMode = flag.Bool("standalone", false, "Run in standalone tapper and API mode") var apiServerAddress = flag.String("api-server-address", "", "Address of mizu API server") var namespace = flag.String("namespace", "", "Resolve IPs if they belong to resources in this namespace (default is all)") +var harsReaderMode = flag.Bool("hars-read", false, "Run in hars-read mode") +var harsDir = flag.String("hars-dir", "", "Directory to read hars from") func main() { flag.Parse() hostMode := os.Getenv(shared.HostModeEnvVar) == "1" tapOpts := &tap.TapOpts{HostMode: hostMode} - if !*tapperMode && !*apiServerMode && !*standaloneMode { - panic("One of the flags --tap, --api or --standalone must be provided") + + if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode{ + panic("One of the flags --tap, --api or --standalone or --hars-read must be provided") } if *standaloneMode { @@ -77,6 +80,13 @@ func main() { go api.StartReadingEntries(filteredHarChannel, nil) hostApi(socketHarOutChannel) + } else if *harsReaderMode { + socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000) + filteredHarChannel := make(chan *tap.OutputChannelItem) + + go filterHarItems(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions()) + go api.StartReadingEntries(filteredHarChannel, harsDir) + hostApi(nil) } signalChan := make(chan os.Signal, 1) @@ -149,15 +159,13 @@ func getTrafficFilteringOptions() *shared.TrafficFilteringOptions { return &filteringOptions } -var userAgentsToFilter = []string{"kube-probe", "prometheus"} - func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) { for message := range inChannel { if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) { continue } // TODO: move this to tappers https://up9.atlassian.net/browse/TRA-3441 - if filterOptions.HideHealthChecks && isHealthCheckByUserAgent(message) { + if isHealthCheckByUserAgent(message, filterOptions.HealthChecksUserAgentHeaders) { continue } @@ -169,11 +177,11 @@ func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *ta } } -func isHealthCheckByUserAgent(message *tap.OutputChannelItem) bool { +func isHealthCheckByUserAgent(message *tap.OutputChannelItem, userAgentsToIgnore []string) bool { for _, header := range message.HarEntry.Request.Headers { if strings.ToLower(header.Name) == "user-agent" { - for _, userAgent := range userAgentsToFilter { - if strings.Contains(strings.ToLower(header.Value), userAgent) { + for _, userAgent := range userAgentsToIgnore { + if strings.Contains(strings.ToLower(header.Value), strings.ToLower(userAgent)) { return true } } diff --git a/agent/pkg/api/main.go b/agent/pkg/api/main.go index 942f09569..90a411ee2 100644 --- a/agent/pkg/api/main.go +++ b/agent/pkg/api/main.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "mizuserver/pkg/holder" + "mizuserver/pkg/providers" "net/url" "os" "path" @@ -108,6 +109,7 @@ func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) { } for item := range outputItems { + providers.EntryAdded() saveHarToDb(item.HarEntry, item.ConnectionInfo) } } diff --git a/agent/pkg/controllers/entries_controller.go b/agent/pkg/controllers/entries_controller.go index a1ad22d42..99bb8d8fb 100644 --- a/agent/pkg/controllers/entries_controller.go +++ b/agent/pkg/controllers/entries_controller.go @@ -241,14 +241,7 @@ func DeleteAllEntries(c *gin.Context) { } func GetGeneralStats(c *gin.Context) { - sqlQuery := "SELECT count(*) as count, min(timestamp) as min, max(timestamp) as max from mizu_entries" - var result struct { - Count int - Min int - Max int - } - database.GetEntriesTable().Raw(sqlQuery).Scan(&result) - c.JSON(http.StatusOK, result) + c.JSON(http.StatusOK, providers.GetGeneralStats()) } func GetTappingStatus(c *gin.Context) { diff --git a/agent/pkg/models/models.go b/agent/pkg/models/models.go index 0648d4f5c..bf8c0cb7b 100644 --- a/agent/pkg/models/models.go +++ b/agent/pkg/models/models.go @@ -56,12 +56,14 @@ type BaseEntryDetails struct { type ApplicableRules struct { Latency int64 `json:"latency,omitempty"` Status bool `json:"status,omitempty"` + NumberOfRules int `json:"numberOfRules,omitempty"` } -func NewApplicableRules(status bool, latency int64) ApplicableRules { +func NewApplicableRules(status bool, latency int64, number int) ApplicableRules { ar := ApplicableRules{} ar.Status = status ar.Latency = latency + ar.NumberOfRules = number return ar } @@ -218,7 +220,7 @@ func (fewp *FullEntryWithPolicy) UnmarshalData(entry *MizuEntry) error { func RunValidationRulesState(harEntry har.Entry, service string) ApplicableRules { numberOfRules, resultPolicyToSend := rules.MatchRequestPolicy(harEntry, service) - statusPolicyToSend, latency := rules.PassedValidationRules(resultPolicyToSend, numberOfRules) - ar := NewApplicableRules(statusPolicyToSend, latency) + statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend, numberOfRules) + ar := NewApplicableRules(statusPolicyToSend, latency, numberOfRules) return ar } diff --git a/agent/pkg/providers/stats_provider.go b/agent/pkg/providers/stats_provider.go new file mode 100644 index 000000000..0f42f0fb8 --- /dev/null +++ b/agent/pkg/providers/stats_provider.go @@ -0,0 +1,36 @@ +package providers + +import ( + "reflect" + "time" +) + +type GeneralStats struct { + EntriesCount int + FirstEntryTimestamp int + LastEntryTimestamp int +} + +var generalStats = GeneralStats{} + +func ResetGeneralStats() { + generalStats = GeneralStats{} +} + +func GetGeneralStats() GeneralStats { + return generalStats +} + +func EntryAdded() { + generalStats.EntriesCount++ + + currentTimestamp := int(time.Now().Unix()) + + if reflect.Value.IsZero(reflect.ValueOf(generalStats.FirstEntryTimestamp)) { + generalStats.FirstEntryTimestamp = currentTimestamp + } + + generalStats.LastEntryTimestamp = currentTimestamp +} + + diff --git a/agent/pkg/providers/stats_provider_test.go b/agent/pkg/providers/stats_provider_test.go new file mode 100644 index 000000000..13acfece9 --- /dev/null +++ b/agent/pkg/providers/stats_provider_test.go @@ -0,0 +1,35 @@ +package providers_test + +import ( + "fmt" + "mizuserver/pkg/providers" + "testing" +) + +func TestNoEntryAddedCount(t *testing.T) { + entriesStats := providers.GetGeneralStats() + + if entriesStats.EntriesCount != 0 { + t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesCount) + } +} + +func TestEntryAddedCount(t *testing.T) { + tests := []int{1, 5, 10, 100, 500, 1000} + + for _, entriesCount := range tests { + t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) { + for i := 0; i < entriesCount; i++ { + providers.EntryAdded() + } + + entriesStats := providers.GetGeneralStats() + + if entriesStats.EntriesCount != entriesCount { + t.Errorf("unexpected result - expected: %v, actual: %v", entriesCount, entriesStats.EntriesCount) + } + + t.Cleanup(providers.ResetGeneralStats) + }) + } +} diff --git a/agent/pkg/rules/models.go b/agent/pkg/rules/models.go index 8b1f1f617..2107e6447 100644 --- a/agent/pkg/rules/models.go +++ b/agent/pkg/rules/models.go @@ -92,19 +92,19 @@ func MatchRequestPolicy(harEntry har.Entry, service string) (int, []RulesMatched return len(enforcePolicy.Rules), resultPolicyToSend } -func PassedValidationRules(rulesMatched []RulesMatched, numberOfRules int) (bool, int64) { +func PassedValidationRules(rulesMatched []RulesMatched, numberOfRules int) (bool, int64, int) { if len(rulesMatched) == 0 { - return false, 0 + return false, 0, 0 } for _, rule := range rulesMatched { if rule.Matched == false { - return false, -1 + return false, -1, len(rulesMatched) } } for _, rule := range rulesMatched { if strings.ToLower(rule.Rule.Type) == "latency" { - return true, rule.Rule.Latency + return true, rule.Rule.Latency, len(rulesMatched) } } - return true, -1 + return true, -1, len(rulesMatched) } diff --git a/agent/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go b/agent/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go index cc0e4d289..4a4889147 100644 --- a/agent/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go +++ b/agent/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go @@ -158,9 +158,11 @@ func filterJsonBody(bytes []byte) ([]byte, error) { func filterJsonMap(jsonMap map[string] interface{}) { for key, value := range jsonMap { + // Do not replace nil values with maskedFieldPlaceholderValue if value == nil { - return + continue } + nestedMap, isNested := value.(map[string] interface{}) if isNested { filterJsonMap(nestedMap) diff --git a/agent/start.sh b/agent/start.sh deleted file mode 100755 index 4b04b6d47..000000000 --- a/agent/start.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -./mizuagent -i any -hardump -targets ${TAPPED_ADDRESSES} diff --git a/assets/validation-example1.png b/assets/validation-example1.png new file mode 100644 index 000000000..cac14b6d2 Binary files /dev/null and b/assets/validation-example1.png differ diff --git a/assets/validation-example2.png b/assets/validation-example2.png new file mode 100644 index 000000000..d870c39e2 Binary files /dev/null and b/assets/validation-example2.png differ diff --git a/build-agent-ci.sh b/build-agent-ci.sh new file mode 100755 index 000000000..430177a9c --- /dev/null +++ b/build-agent-ci.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +GCP_PROJECT=up9-docker-hub +REPOSITORY=gcr.io/$GCP_PROJECT +SERVER_NAME=mizu +GIT_BRANCH=ci + +DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH +SEM_VER=${SEM_VER=0.0.0} + +DOCKER_TAGGED_BUILD="$DOCKER_REPO:$SEM_VER" + +echo "building $DOCKER_TAGGED_BUILD" +docker build -t ${DOCKER_TAGGED_BUILD} --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} . diff --git a/build-push-featurebranch.sh b/build-push-featurebranch.sh index 0441c3960..4e040ac69 100755 --- a/build-push-featurebranch.sh +++ b/build-push-featurebranch.sh @@ -1,12 +1,14 @@ #!/bin/bash set -e -SERVER_NAME=mizu GCP_PROJECT=up9-docker-hub REPOSITORY=gcr.io/$GCP_PROJECT +SERVER_NAME=mizu GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]') -SEM_VER=${SEM_VER=0.0.0} + DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH +SEM_VER=${SEM_VER=0.0.0} + DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER") if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ] @@ -21,6 +23,6 @@ docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_ for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}" do - echo pushing "$DOCKER_TAG" - docker push "$DOCKER_TAG" + echo pushing "$DOCKER_TAG" + docker push "$DOCKER_TAG" done diff --git a/cli/Makefile b/cli/Makefile index b4841ba0c..dd77c510d 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -24,7 +24,7 @@ build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables build-all: ## Build for all supported platforms. @echo "Compiling for every OS and Platform" - @mkdir -p bin && echo "SHA256 checksums available for compiled binaries \n\nRun \`shasum -a 256 -c mizu_OS_ARCH.sha256\` to verify\n\n" > bin/README.md + @mkdir -p bin && sed s/_SEM_VER_/$(SEM_VER)/g README.md.TEMPLATE > bin/README.md @$(MAKE) build GOOS=darwin GOARCH=amd64 @$(MAKE) build GOOS=linux GOARCH=amd64 @# $(MAKE) build GOOS=darwin GOARCH=arm64 @@ -39,3 +39,6 @@ build-all: ## Build for all supported platforms. clean: ## Clean all build artifacts. go clean rm -rf ./bin/* + +test: ## Run cli tests. + @go test ./... -coverpkg=./... -race -coverprofile=coverage.out -covermode=atomic diff --git a/cli/README.md.TEMPLATE b/cli/README.md.TEMPLATE new file mode 100644 index 000000000..83288847f --- /dev/null +++ b/cli/README.md.TEMPLATE @@ -0,0 +1,20 @@ +# Mizu release _SEM_VER_ + +Download Mizu for your platform + +**Mac** (on Intel chip) +``` +curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_darwin_amd64 && chmod 755 mizu +``` + +**Linux** +``` +curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_linux_amd64 && chmod 755 mizu +``` + + +### Checksums +SHA256 checksums available for compiled binaries. +Run `shasum -a 256 -c mizu_OS_ARCH.sha256` to verify. + + diff --git a/cli/cmd/config.go b/cli/cmd/config.go index 66d4485d8..2b30a9630 100644 --- a/cli/cmd/config.go +++ b/cli/cmd/config.go @@ -2,32 +2,36 @@ package cmd import ( "fmt" + "github.com/creasty/defaults" "github.com/spf13/cobra" - "github.com/up9inc/mizu/cli/mizu" + "github.com/up9inc/mizu/cli/config" + "github.com/up9inc/mizu/cli/config/configStructs" + "github.com/up9inc/mizu/cli/logger" + "github.com/up9inc/mizu/cli/telemetry" "github.com/up9inc/mizu/cli/uiUtils" "io/ioutil" ) -var regenerateFile bool - var configCmd = &cobra.Command{ Use: "config", Short: "Generate config with default values", RunE: func(cmd *cobra.Command, args []string) error { - template, err := mizu.GetConfigWithDefaults() + go telemetry.ReportRun("config", config.Config.Config) + + template, err := config.GetConfigWithDefaults() if err != nil { - mizu.Log.Errorf("Failed generating config with defaults %v", err) + logger.Log.Errorf("Failed generating config with defaults %v", err) return nil } - if regenerateFile { + if config.Config.Config.Regenerate { data := []byte(template) - if err := ioutil.WriteFile(mizu.GetConfigFilePath(), data, 0644); err != nil { - mizu.Log.Errorf("Failed writing config %v", err) + if err := ioutil.WriteFile(config.GetConfigFilePath(), data, 0644); err != nil { + logger.Log.Errorf("Failed writing config %v", err) return nil } - mizu.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, mizu.GetConfigFilePath()))) + logger.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, config.GetConfigFilePath()))) } else { - mizu.Log.Debugf("Writing template config.\n%v", template) + logger.Log.Debugf("Writing template config.\n%v", template) fmt.Printf("%v", template) } return nil @@ -36,5 +40,9 @@ var configCmd = &cobra.Command{ func init() { rootCmd.AddCommand(configCmd) - configCmd.Flags().BoolVarP(®enerateFile, "regenerate", "r", false, fmt.Sprintf("Regenerate the config file with default values %s", mizu.GetConfigFilePath())) + + defaultConfigConfig := configStructs.ConfigConfig{} + defaults.Set(&defaultConfigConfig) + + configCmd.Flags().BoolP(configStructs.RegenerateConfigName, "r", defaultConfigConfig.Regenerate, fmt.Sprintf("Regenerate the config file with default values %s", config.GetConfigFilePath())) } diff --git a/cli/cmd/fetch.go b/cli/cmd/fetch.go index 3b9b4cc09..f18e5bf3f 100644 --- a/cli/cmd/fetch.go +++ b/cli/cmd/fetch.go @@ -3,16 +3,19 @@ package cmd import ( "github.com/creasty/defaults" "github.com/spf13/cobra" - "github.com/up9inc/mizu/cli/mizu" - "github.com/up9inc/mizu/cli/mizu/configStructs" + "github.com/up9inc/mizu/cli/config" + "github.com/up9inc/mizu/cli/config/configStructs" + "github.com/up9inc/mizu/cli/mizu/version" + "github.com/up9inc/mizu/cli/telemetry" ) var fetchCmd = &cobra.Command{ Use: "fetch", Short: "Download recorded traffic to files", RunE: func(cmd *cobra.Command, args []string) error { - go mizu.ReportRun("fetch", mizu.Config.Fetch) - if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.Fetch.GuiPort); err != nil { + go telemetry.ReportRun("fetch", config.Config.Fetch) + + if isCompatible, err := version.CheckVersionCompatibility(config.Config.Fetch.GuiPort); err != nil { return err } else if !isCompatible { return nil diff --git a/cli/cmd/fetchRunner.go b/cli/cmd/fetchRunner.go index 9c372a632..b0c76020e 100644 --- a/cli/cmd/fetchRunner.go +++ b/cli/cmd/fetchRunner.go @@ -4,8 +4,9 @@ import ( "archive/zip" "bytes" "fmt" + "github.com/up9inc/mizu/cli/config" "github.com/up9inc/mizu/cli/kubernetes" - "github.com/up9inc/mizu/cli/mizu" + "github.com/up9inc/mizu/cli/logger" "io" "io/ioutil" "log" @@ -16,8 +17,8 @@ import ( ) func RunMizuFetch() { - mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Fetch.GuiPort) - resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, mizu.Config.Fetch.FromTimestamp, mizu.Config.Fetch.ToTimestamp)) + mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Fetch.GuiPort) + resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, config.Config.Fetch.FromTimestamp, config.Config.Fetch.ToTimestamp)) if err != nil { log.Fatal(err) } @@ -34,7 +35,7 @@ func RunMizuFetch() { log.Fatal(err) } - _ = Unzip(zipReader, mizu.Config.Fetch.Directory) + _ = Unzip(zipReader, config.Config.Fetch.Directory) } func Unzip(reader *zip.Reader, dest string) error { @@ -64,7 +65,7 @@ func Unzip(reader *zip.Reader, dest string) error { _ = os.MkdirAll(path, f.Mode()) } else { _ = os.MkdirAll(filepath.Dir(path), f.Mode()) - mizu.Log.Infof("writing HAR file [ %v ]", path) + logger.Log.Infof("writing HAR file [ %v ]", path) f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) if err != nil { return err @@ -73,7 +74,7 @@ func Unzip(reader *zip.Reader, dest string) error { if err := f.Close(); err != nil { panic(err) } - mizu.Log.Info(" done") + logger.Log.Info(" done") }() _, err = io.Copy(f, rc) diff --git a/cli/cmd/logs.go b/cli/cmd/logs.go index ec639fc7e..16c626f33 100644 --- a/cli/cmd/logs.go +++ b/cli/cmd/logs.go @@ -2,38 +2,38 @@ package cmd import ( "context" + "github.com/creasty/defaults" "github.com/spf13/cobra" - "github.com/up9inc/mizu/cli/fsUtils" + "github.com/up9inc/mizu/cli/config" + "github.com/up9inc/mizu/cli/config/configStructs" + "github.com/up9inc/mizu/cli/errormessage" "github.com/up9inc/mizu/cli/kubernetes" - "github.com/up9inc/mizu/cli/mizu" - "os" - "path" + "github.com/up9inc/mizu/cli/logger" + "github.com/up9inc/mizu/cli/mizu/fsUtils" + "github.com/up9inc/mizu/cli/telemetry" ) -var filePath string - var logsCmd = &cobra.Command{ Use: "logs", Short: "Create a zip file with logs for Github issue or troubleshoot", RunE: func(cmd *cobra.Command, args []string) error { - kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.View.KubeConfigPath) + go telemetry.ReportRun("logs", config.Config.Logs) + + kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath()) if err != nil { + logger.Log.Error(err) return nil } ctx, _ := context.WithCancel(context.Background()) - if filePath == "" { - pwd, err := os.Getwd() - if err != nil { - mizu.Log.Errorf("Failed to get PWD, %v (try using `mizu logs -f )`", err) - return nil - } - filePath = path.Join(pwd, "mizu_logs.zip") + if validationErr := config.Config.Logs.Validate(); validationErr != nil { + return errormessage.FormatError(validationErr) } - mizu.Log.Debugf("Using file path %s", filePath) - if err := fsUtils.DumpLogs(kubernetesProvider, ctx, filePath); err != nil { - mizu.Log.Errorf("Failed dump logs %v", err) + logger.Log.Debugf("Using file path %s", config.Config.Logs.FilePath()) + + if dumpLogsErr := fsUtils.DumpLogs(kubernetesProvider, ctx, config.Config.Logs.FilePath()); dumpLogsErr != nil { + logger.Log.Errorf("Failed dump logs %v", dumpLogsErr) } return nil @@ -42,5 +42,9 @@ var logsCmd = &cobra.Command{ func init() { rootCmd.AddCommand(logsCmd) - logsCmd.Flags().StringVarP(&filePath, "file", "f", "", "Path for zip file (default current \\mizu_logs.zip)") + + defaultLogsConfig := configStructs.LogsConfig{} + defaults.Set(&defaultLogsConfig) + + logsCmd.Flags().StringP(configStructs.FileLogsName, "f", defaultLogsConfig.FileStr, "Path for zip file (default current \\mizu_logs.zip)") } diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 84f6dde3b..3a1f4d3e0 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -3,8 +3,12 @@ package cmd import ( "fmt" "github.com/spf13/cobra" - "github.com/up9inc/mizu/cli/fsUtils" + "github.com/up9inc/mizu/cli/config" + "github.com/up9inc/mizu/cli/logger" "github.com/up9inc/mizu/cli/mizu" + "github.com/up9inc/mizu/cli/mizu/fsUtils" + "github.com/up9inc/mizu/cli/mizu/version" + "github.com/up9inc/mizu/cli/uiUtils" ) var rootCmd = &cobra.Command{ @@ -13,24 +17,35 @@ var rootCmd = &cobra.Command{ Long: `A web traffic viewer for kubernetes Further info is available at https://github.com/up9inc/mizu`, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := fsUtils.EnsureDir(mizu.GetMizuFolderPath()); err != nil { - mizu.Log.Errorf("Failed to use mizu folder, %v", err) + if err := config.InitConfig(cmd); err != nil { + logger.Log.Fatal(err) } - mizu.InitLogger() - if err := mizu.InitConfig(cmd); err != nil { - mizu.Log.Fatal(err) - } - return nil }, } func init() { - rootCmd.PersistentFlags().StringSlice(mizu.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", mizu.SetCommandName)) + rootCmd.PersistentFlags().StringSlice(config.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", config.SetCommandName)) +} + +func printNewVersionIfNeeded(versionChan chan string) { + versionMsg := <-versionChan + if versionMsg != "" { + logger.Log.Infof(uiUtils.Yellow, versionMsg) + } } // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the tapCmd. func Execute() { + if err := fsUtils.EnsureDir(mizu.GetMizuFolderPath()); err != nil { + logger.Log.Errorf("Failed to use mizu folder, %v", err) + } + logger.InitLogger() + + versionChan := make(chan string) + defer printNewVersionIfNeeded(versionChan) + go version.CheckNewerVersion(versionChan) + cobra.CheckErr(rootCmd.Execute()) } diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 72e71f3fb..00e59e90a 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -2,13 +2,15 @@ package cmd import ( "errors" + "github.com/up9inc/mizu/cli/config" + "github.com/up9inc/mizu/cli/config/configStructs" + "github.com/up9inc/mizu/cli/logger" + "github.com/up9inc/mizu/cli/telemetry" "os" "github.com/creasty/defaults" "github.com/spf13/cobra" "github.com/up9inc/mizu/cli/errormessage" - "github.com/up9inc/mizu/cli/mizu" - "github.com/up9inc/mizu/cli/mizu/configStructs" "github.com/up9inc/mizu/cli/uiUtils" ) @@ -20,31 +22,27 @@ var tapCmd = &cobra.Command{ Long: `Record the ingoing traffic of a kubernetes pod. Supported protocols are HTTP and gRPC.`, RunE: func(cmd *cobra.Command, args []string) error { - go mizu.ReportRun("tap", mizu.Config.Tap) + go telemetry.ReportRun("tap", config.Config.Tap) RunMizuTap() return nil }, PreRunE: func(cmd *cobra.Command, args []string) error { if len(args) == 1 { - mizu.Config.Tap.PodRegexStr = args[0] + config.Config.Tap.PodRegexStr = args[0] } else if len(args) > 1 { return errors.New("unexpected number of arguments") } - if err := mizu.Config.Validate(); err != nil { + if err := config.Config.Tap.Validate(); err != nil { return errormessage.FormatError(err) } - if err := mizu.Config.Tap.Validate(); err != nil { - return errormessage.FormatError(err) - } + logger.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", config.Config.Tap.HumanMaxEntriesDBSize) - mizu.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", mizu.Config.Tap.HumanMaxEntriesDBSize) - - if mizu.Config.Tap.Analysis { - mizu.Log.Infof(analysisMessageToConfirm) + if config.Config.Tap.Analysis { + logger.Log.Infof(analysisMessageToConfirm) if !uiUtils.AskForConfirmation("Would you like to proceed [Y/n]: ") { - mizu.Log.Infof("You can always run mizu without analysis, aborting") + logger.Log.Infof("You can always run mizu without analysis, aborting") os.Exit(0) } } @@ -60,11 +58,10 @@ func init() { defaults.Set(&defaultTapConfig) tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver") - tapCmd.Flags().StringArrayP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector") + tapCmd.Flags().StringSliceP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector") tapCmd.Flags().Bool(configStructs.AnalysisTapName, defaultTapConfig.Analysis, "Uploads traffic to UP9 for further analysis (Beta)") tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces") - tapCmd.Flags().StringArrayP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies") - tapCmd.Flags().Bool(configStructs.HideHealthChecksTapName, defaultTapConfig.HideHealthChecks, "Hides requests with kube-probe or prometheus user-agent headers") + tapCmd.Flags().StringSliceP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies") tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values") tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size") tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any") diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 401b7f77d..762c1222f 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -5,9 +5,12 @@ import ( "context" "encoding/json" "fmt" - "github.com/up9inc/mizu/cli/fsUtils" - "github.com/up9inc/mizu/cli/goUtils" - "github.com/up9inc/mizu/cli/mizu/configStructs" + "github.com/up9inc/mizu/cli/config" + "github.com/up9inc/mizu/cli/config/configStructs" + "github.com/up9inc/mizu/cli/logger" + "github.com/up9inc/mizu/cli/mizu/fsUtils" + "github.com/up9inc/mizu/cli/mizu/goUtils" + "github.com/up9inc/mizu/cli/telemetry" "net/http" "net/url" "os" @@ -46,21 +49,21 @@ var state tapState func RunMizuTap() { mizuApiFilteringOptions, err := getMizuApiFilteringOptions() if err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error parsing regex-masking: %v", errormessage.FormatError(err))) + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error parsing regex-masking: %v", errormessage.FormatError(err))) return } var mizuValidationRules string - if mizu.Config.Tap.EnforcePolicyFile != "" { - mizuValidationRules, err = readValidationRules(mizu.Config.Tap.EnforcePolicyFile) + if config.Config.Tap.EnforcePolicyFile != "" { + mizuValidationRules, err = readValidationRules(config.Config.Tap.EnforcePolicyFile) if err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading policy file: %v", errormessage.FormatError(err))) + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading policy file: %v", errormessage.FormatError(err))) return } } - kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.KubeConfigPath) + kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath()) if err != nil { - mizu.Log.Error(err) + logger.Log.Error(err) return } @@ -69,37 +72,45 @@ func RunMizuTap() { targetNamespaces := getNamespaces(kubernetesProvider) + if config.Config.IsNsRestrictedMode() { + if len(targetNamespaces) != 1 || !mizu.Contains(targetNamespaces, config.Config.MizuResourcesNamespace) { + logger.Log.Errorf("Not supported mode. Mizu can't resolve IPs in other namespaces when running in namespace restricted mode.\n"+ + "You can use the same namespace for --%s and --%s", configStructs.NamespacesTapName, config.MizuResourcesNamespaceConfigName) + return + } + } + var namespacesStr string - if targetNamespaces[0] != mizu.K8sAllNamespaces { + if !mizu.Contains(targetNamespaces, mizu.K8sAllNamespaces) { namespacesStr = fmt.Sprintf("namespaces \"%s\"", strings.Join(targetNamespaces, "\", \"")) } else { namespacesStr = "all namespaces" } - mizu.CheckNewerVersion() - mizu.Log.Infof("Tapping pods in %s", namespacesStr) + + logger.Log.Infof("Tapping pods in %s", namespacesStr) if err, _ := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespaces); err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error getting pods by regex: %v", errormessage.FormatError(err))) + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error getting pods by regex: %v", errormessage.FormatError(err))) return } if len(state.currentlyTappedPods) == 0 { var suggestionStr string - if targetNamespaces[0] != mizu.K8sAllNamespaces { + if !mizu.Contains(targetNamespaces, mizu.K8sAllNamespaces) { suggestionStr = ". Select a different namespace with -n or tap all namespaces with -A" } - mizu.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any pods matching the regex argument%s", suggestionStr)) + logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any pods matching the regex argument%s", suggestionStr)) } - if mizu.Config.Tap.DryRun { + if config.Config.Tap.DryRun { return } nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods) - defer cleanUpMizuResources(kubernetesProvider) + defer cleanUpMizu(kubernetesProvider) if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions, mizuValidationRules); err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err))) + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err))) return } @@ -120,7 +131,7 @@ func readValidationRules(file string) (string, error) { } func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, mizuApiFilteringOptions *shared.TrafficFilteringOptions, mizuValidationRules string) error { - if !mizu.Config.IsNsRestrictedMode() { + if !config.Config.IsNsRestrictedMode() { if err := createMizuNamespace(ctx, kubernetesProvider); err != nil { return err } @@ -135,7 +146,7 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro } if err := createMizuConfigmap(ctx, kubernetesProvider, mizuValidationRules); err != nil { - mizu.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err))) + logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err))) state.doNotRemoveConfigMap = true } else if mizuValidationRules == "" { state.doNotRemoveConfigMap = true @@ -145,12 +156,12 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro } func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, data string) error { - err := kubernetesProvider.CreateConfigMap(ctx, mizu.Config.MizuResourcesNamespace, mizu.ConfigMapName, data) + err := kubernetesProvider.CreateConfigMap(ctx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName, data) return err } func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Provider) error { - _, err := kubernetesProvider.CreateNamespace(ctx, mizu.Config.MizuResourcesNamespace) + _, err := kubernetesProvider.CreateNamespace(ctx, config.Config.MizuResourcesNamespace) return err } @@ -159,7 +170,7 @@ func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Pro state.mizuServiceAccountExists, err = createRBACIfNecessary(ctx, kubernetesProvider) if err != nil { - mizu.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err))) + logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err))) } var serviceAccountName string @@ -170,25 +181,27 @@ func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Pro } opts := &kubernetes.ApiServerOptions{ - Namespace: mizu.Config.MizuResourcesNamespace, + Namespace: config.Config.MizuResourcesNamespace, PodName: mizu.ApiServerPodName, - PodImage: mizu.Config.AgentImage, + PodImage: config.Config.AgentImage, ServiceAccountName: serviceAccountName, - IsNamespaceRestricted: mizu.Config.IsNsRestrictedMode(), + IsNamespaceRestricted: config.Config.IsNsRestrictedMode(), MizuApiFilteringOptions: mizuApiFilteringOptions, - MaxEntriesDBSizeBytes: mizu.Config.Tap.MaxEntriesDBSizeBytes(), + MaxEntriesDBSizeBytes: config.Config.Tap.MaxEntriesDBSizeBytes(), + Resources: config.Config.Tap.ApiServerResources, + ImagePullPolicy: config.Config.ImagePullPolicy(), } _, err = kubernetesProvider.CreateMizuApiServerPod(ctx, opts) if err != nil { return err } - mizu.Log.Debugf("Successfully created API server pod: %s", mizu.ApiServerPodName) + logger.Log.Debugf("Successfully created API server pod: %s", mizu.ApiServerPodName) - state.apiServerService, err = kubernetesProvider.CreateService(ctx, mizu.Config.MizuResourcesNamespace, mizu.ApiServerPodName, mizu.ApiServerPodName) + state.apiServerService, err = kubernetesProvider.CreateService(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName, mizu.ApiServerPodName) if err != nil { return err } - mizu.Log.Debugf("Successfully created service: %s", mizu.ApiServerPodName) + logger.Log.Debugf("Successfully created service: %s", mizu.ApiServerPodName) return nil } @@ -196,9 +209,9 @@ func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Pro func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) { var compiledRegexSlice []*shared.SerializableRegexp - if mizu.Config.Tap.PlainTextFilterRegexes != nil && len(mizu.Config.Tap.PlainTextFilterRegexes) > 0 { + if config.Config.Tap.PlainTextFilterRegexes != nil && len(config.Config.Tap.PlainTextFilterRegexes) > 0 { compiledRegexSlice = make([]*shared.SerializableRegexp, 0) - for _, regexStr := range mizu.Config.Tap.PlainTextFilterRegexes { + for _, regexStr := range config.Config.Tap.PlainTextFilterRegexes { compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr) if err != nil { return nil, err @@ -207,7 +220,11 @@ func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) { } } - return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: mizu.Config.Tap.HideHealthChecks, DisableRedaction: mizu.Config.Tap.DisableRedaction}, nil + return &shared.TrafficFilteringOptions{ + PlainTextMaskingRegexes: compiledRegexSlice, + HealthChecksUserAgentHeaders: config.Config.Tap.HealthChecksUserAgentHeaders, + DisableRedaction: config.Config.Tap.DisableRedaction, + }, nil } func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string) error { @@ -221,20 +238,22 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi if err := kubernetesProvider.ApplyMizuTapperDaemonSet( ctx, - mizu.Config.MizuResourcesNamespace, + config.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName, - mizu.Config.AgentImage, + config.Config.AgentImage, mizu.TapperPodName, fmt.Sprintf("%s.%s.svc.cluster.local", state.apiServerService.Name, state.apiServerService.Namespace), nodeToTappedPodIPMap, serviceAccountName, - mizu.Config.Tap.TapOutgoing(), + config.Config.Tap.TapOutgoing(), + config.Config.Tap.TapperResources, + config.Config.ImagePullPolicy(), ); err != nil { return err } - mizu.Log.Debugf("Successfully created %v tappers", len(nodeToTappedPodIPMap)) + logger.Log.Debugf("Successfully created %v tappers", len(nodeToTappedPodIPMap)) } else { - if err := kubernetesProvider.RemoveDaemonSet(ctx, mizu.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil { + if err := kubernetesProvider.RemoveDaemonSet(ctx, config.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil { return err } } @@ -242,70 +261,74 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi return nil } -func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) { +func cleanUpMizu(kubernetesProvider *kubernetes.Provider) { + telemetry.ReportAPICalls(config.Config.Tap.GuiPort) + cleanUpMizuResources(kubernetesProvider) +} +func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) { removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) defer cancel() - if mizu.Config.DumpLogs { + if config.Config.DumpLogs { mizuDir := mizu.GetMizuFolderPath() - filePath = path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05"))) + filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05"))) if err := fsUtils.DumpLogs(kubernetesProvider, removalCtx, filePath); err != nil { - mizu.Log.Errorf("Failed dump logs %v", err) + logger.Log.Errorf("Failed dump logs %v", err) } } - mizu.Log.Infof("\nRemoving mizu resources\n") + logger.Log.Infof("\nRemoving mizu resources\n") - if !mizu.Config.IsNsRestrictedMode() { - if err := kubernetesProvider.RemoveNamespace(removalCtx, mizu.Config.MizuResourcesNamespace); err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Namespace %s: %v", mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err))) + if !config.Config.IsNsRestrictedMode() { + if err := kubernetesProvider.RemoveNamespace(removalCtx, config.Config.MizuResourcesNamespace); err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Namespace %s: %v", config.Config.MizuResourcesNamespace, errormessage.FormatError(err))) return } } else { - if err := kubernetesProvider.RemovePod(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Pod %s in namespace %s: %v", mizu.ApiServerPodName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err))) + if err := kubernetesProvider.RemovePod(removalCtx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Pod %s in namespace %s: %v", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err))) } - if err := kubernetesProvider.RemoveService(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service %s in namespace %s: %v", mizu.ApiServerPodName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err))) + if err := kubernetesProvider.RemoveService(removalCtx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service %s in namespace %s: %v", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err))) } - if err := kubernetesProvider.RemoveDaemonSet(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing DaemonSet %s in namespace %s: %v", mizu.TapperDaemonSetName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err))) + if err := kubernetesProvider.RemoveDaemonSet(removalCtx, config.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing DaemonSet %s in namespace %s: %v", mizu.TapperDaemonSetName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err))) } if !state.doNotRemoveConfigMap { - if err := kubernetesProvider.RemoveConfigMap(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.ConfigMapName); err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing ConfigMap %s in namespace %s: %v", mizu.ConfigMapName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err))) + if err := kubernetesProvider.RemoveConfigMap(removalCtx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName); err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing ConfigMap %s in namespace %s: %v", mizu.ConfigMapName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err))) } } } if state.mizuServiceAccountExists { - if !mizu.Config.IsNsRestrictedMode() { + if !config.Config.IsNsRestrictedMode() { if err := kubernetesProvider.RemoveNonNamespacedResources(removalCtx, mizu.ClusterRoleName, mizu.ClusterRoleBindingName); err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing non-namespaced resources: %v", errormessage.FormatError(err))) + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing non-namespaced resources: %v", errormessage.FormatError(err))) return } } else { - if err := kubernetesProvider.RemoveServicAccount(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.ServiceAccountName); err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service Account %s in namespace %s: %v", mizu.ServiceAccountName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err))) + if err := kubernetesProvider.RemoveServicAccount(removalCtx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName); err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service Account %s in namespace %s: %v", mizu.ServiceAccountName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err))) return } - if err := kubernetesProvider.RemoveRole(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.RoleName); err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Role %s in namespace %s: %v", mizu.RoleName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err))) + if err := kubernetesProvider.RemoveRole(removalCtx, config.Config.MizuResourcesNamespace, mizu.RoleName); err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Role %s in namespace %s: %v", mizu.RoleName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err))) } - if err := kubernetesProvider.RemoveRoleBinding(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.RoleBindingName); err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing RoleBinding %s in namespace %s: %v", mizu.RoleBindingName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err))) + if err := kubernetesProvider.RemoveRoleBinding(removalCtx, config.Config.MizuResourcesNamespace, mizu.RoleBindingName); err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing RoleBinding %s in namespace %s: %v", mizu.RoleBindingName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err))) } } } - if !mizu.Config.IsNsRestrictedMode() { + if !config.Config.IsNsRestrictedMode() { waitUntilNamespaceDeleted(removalCtx, cancel, kubernetesProvider) } } @@ -316,20 +339,20 @@ func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, k waitForFinish(ctx, cancel) }() - if err := kubernetesProvider.WaitUtilNamespaceDeleted(ctx, mizu.Config.MizuResourcesNamespace); err != nil { + if err := kubernetesProvider.WaitUtilNamespaceDeleted(ctx, config.Config.MizuResourcesNamespace); err != nil { switch { case ctx.Err() == context.Canceled: // Do nothing. User interrupted the wait. case err == wait.ErrWaitTimeout: - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Timeout while removing Namespace %s", mizu.Config.MizuResourcesNamespace)) + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Timeout while removing Namespace %s", config.Config.MizuResourcesNamespace)) default: - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error while waiting for Namespace %s to be deleted: %v", mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err))) + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error while waiting for Namespace %s to be deleted: %v", config.Config.MizuResourcesNamespace, errormessage.FormatError(err))) } } } func reportTappedPods() { - mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort) + mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort) tappedPodsUrl := fmt.Sprintf("http://%s/status/tappedPods", mizuProxiedUrl) podInfos := make([]shared.PodInfo, 0) @@ -339,30 +362,30 @@ func reportTappedPods() { tapStatus := shared.TapStatus{Pods: podInfos} if jsonValue, err := json.Marshal(tapStatus); err != nil { - mizu.Log.Debugf("[ERROR] failed Marshal the tapped pods %v", err) + logger.Log.Debugf("[ERROR] failed Marshal the tapped pods %v", err) } else { if response, err := http.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil { - mizu.Log.Debugf("[ERROR] failed sending to API server the tapped pods %v", err) + logger.Log.Debugf("[ERROR] failed sending to API server the tapped pods %v", err) } else if response.StatusCode != 200 { - mizu.Log.Debugf("[ERROR] failed sending to API server the tapped pods, response status code %v", response.StatusCode) + logger.Log.Debugf("[ERROR] failed sending to API server the tapped pods, response status code %v", response.StatusCode) } else { - mizu.Log.Debugf("Reported to server API about %d taped pods successfully", len(podInfos)) + logger.Log.Debugf("Reported to server API about %d taped pods successfully", len(podInfos)) } } } func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, targetNamespaces []string, cancel context.CancelFunc) { - added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, targetNamespaces, mizu.Config.Tap.PodRegex()) + added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, targetNamespaces, config.Config.Tap.PodRegex()) restartTappers := func() { err, changeFound := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespaces) if err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Failed to update currently tapped pods: %v", err)) + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Failed to update currently tapped pods: %v", err)) cancel() } if !changeFound { - mizu.Log.Debugf("Nothing changed update tappers not needed") + logger.Log.Debugf("Nothing changed update tappers not needed") return } @@ -370,11 +393,11 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods) if err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error building node to ips map: %v", errormessage.FormatError(err))) + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error building node to ips map: %v", errormessage.FormatError(err))) cancel() } if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating daemonset: %v", errormessage.FormatError(err))) + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating daemonset: %v", errormessage.FormatError(err))) cancel() } } @@ -383,13 +406,13 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro for { select { case pod := <-added: - mizu.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace) + logger.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace) restartTappersDebouncer.SetOn() case pod := <-removed: - mizu.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace) + logger.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace) restartTappersDebouncer.SetOn() case pod := <-modified: - mizu.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP) + logger.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP) // Act only if the modified pod has already obtained an IP address. // After filtering for IPs, on a normal pod restart this includes the following events: // - Pod deletion @@ -401,13 +424,13 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro } case err := <-errorChan: - mizu.Log.Debugf("Watching pods loop, got error %v, stopping `restart tappers debouncer`", err) + logger.Log.Debugf("Watching pods loop, got error %v, stopping `restart tappers debouncer`", err) restartTappersDebouncer.Cancel() // TODO: Does this also perform cleanup? cancel() case <-ctx.Done(): - mizu.Log.Debugf("Watching pods loop, context done, stopping `restart tappers debouncer`") + logger.Log.Debugf("Watching pods loop, context done, stopping `restart tappers debouncer`") restartTappersDebouncer.Cancel() return } @@ -416,18 +439,18 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro func updateCurrentlyTappedPods(kubernetesProvider *kubernetes.Provider, ctx context.Context, targetNamespaces []string) (error, bool) { changeFound := false - if matchingPods, err := kubernetesProvider.ListAllRunningPodsMatchingRegex(ctx, mizu.Config.Tap.PodRegex(), targetNamespaces); err != nil { + if matchingPods, err := kubernetesProvider.ListAllRunningPodsMatchingRegex(ctx, config.Config.Tap.PodRegex(), targetNamespaces); err != nil { return err, false } else { podsToTap := excludeMizuPods(matchingPods) addedPods, removedPods := getPodArrayDiff(state.currentlyTappedPods, podsToTap) for _, addedPod := range addedPods { changeFound = true - mizu.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", addedPod.Name)) + logger.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", addedPod.Name)) } for _, removedPod := range removedPods { changeFound = true - mizu.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s", removedPod.Name)) + logger.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s", removedPod.Name)) } state.currentlyTappedPods = podsToTap } @@ -475,86 +498,86 @@ func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod { func createProxyToApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) { podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.ApiServerPodName)) - added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{mizu.Config.MizuResourcesNamespace}, podExactRegex) + added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex) isPodReady := false timeAfter := time.After(25 * time.Second) for { select { case <-ctx.Done(): - mizu.Log.Debugf("Watching API Server pod loop, ctx done") + logger.Log.Debugf("Watching API Server pod loop, ctx done") return case <-added: - mizu.Log.Debugf("Watching API Server pod loop, added") + logger.Log.Debugf("Watching API Server pod loop, added") continue case <-removed: - mizu.Log.Infof("%s removed", mizu.ApiServerPodName) + logger.Log.Infof("%s removed", mizu.ApiServerPodName) cancel() return case modifiedPod := <-modified: if modifiedPod == nil { - mizu.Log.Debugf("Watching API Server pod loop, modifiedPod with nil") + logger.Log.Debugf("Watching API Server pod loop, modifiedPod with nil") continue } - mizu.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase) + logger.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase) if modifiedPod.Status.Phase == core.PodRunning && !isPodReady { isPodReady = true go startProxyReportErrorIfAny(kubernetesProvider, cancel) - mizu.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort)) + logger.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort)) time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready requestForAnalysis() reportTappedPods() } case <-timeAfter: if !isPodReady { - mizu.Log.Errorf(uiUtils.Error, "Mizu API server was not ready in time") + logger.Log.Errorf(uiUtils.Error, "Mizu API server was not ready in time") cancel() } case <-errorChan: - mizu.Log.Debugf("[ERROR] Agent creation, watching %v namespace", mizu.Config.MizuResourcesNamespace) + logger.Log.Debugf("[ERROR] Agent creation, watching %v namespace", config.Config.MizuResourcesNamespace) cancel() } } } func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) { - err := kubernetes.StartProxy(kubernetesProvider, mizu.Config.Tap.GuiPort, mizu.Config.MizuResourcesNamespace, mizu.ApiServerPodName) + err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName) if err != nil { - mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v\n"+ + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v\n"+ "Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName)) cancel() } } func requestForAnalysis() { - if !mizu.Config.Tap.Analysis { + if !config.Config.Tap.Analysis { return } - mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort) - urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(mizu.Config.Tap.AnalysisDestination), mizu.Config.Tap.SleepIntervalSec) + mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort) + urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(config.Config.Tap.AnalysisDestination), config.Config.Tap.SleepIntervalSec) u, parseErr := url.ParseRequestURI(urlPath) if parseErr != nil { - mizu.Log.Fatal("Failed parsing the URL (consider changing the analysis dest URL), err: %v", parseErr) + logger.Log.Fatal("Failed parsing the URL (consider changing the analysis dest URL), err: %v", parseErr) } - mizu.Log.Debugf("Sending get request to %v", u.String()) + logger.Log.Debugf("Sending get request to %v", u.String()) if response, requestErr := http.Get(u.String()); requestErr != nil { - mizu.Log.Errorf("Failed to notify agent for analysis, err: %v", requestErr) + logger.Log.Errorf("Failed to notify agent for analysis, err: %v", requestErr) } else if response.StatusCode != 200 { - mizu.Log.Errorf("Failed to notify agent for analysis, status code: %v", response.StatusCode) + logger.Log.Errorf("Failed to notify agent for analysis, status code: %v", response.StatusCode) } else { - mizu.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis") + logger.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis") } } func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) (bool, error) { - if !mizu.Config.IsNsRestrictedMode() { - err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.Config.MizuResourcesNamespace, mizu.ServiceAccountName, mizu.ClusterRoleName, mizu.ClusterRoleBindingName, mizu.RBACVersion) + if !config.Config.IsNsRestrictedMode() { + err := kubernetesProvider.CreateMizuRBAC(ctx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName, mizu.ClusterRoleName, mizu.ClusterRoleBindingName, mizu.RBACVersion) if err != nil { return false, err } } else { - err := kubernetesProvider.CreateMizuRBACNamespaceRestricted(ctx, mizu.Config.MizuResourcesNamespace, mizu.ServiceAccountName, mizu.RoleName, mizu.RoleBindingName, mizu.RBACVersion) + err := kubernetesProvider.CreateMizuRBACNamespaceRestricted(ctx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName, mizu.RoleName, mizu.RoleBindingName, mizu.RBACVersion) if err != nil { return false, err } @@ -589,10 +612,10 @@ func waitForFinish(ctx context.Context, cancel context.CancelFunc) { } func getNamespaces(kubernetesProvider *kubernetes.Provider) []string { - if mizu.Config.Tap.AllNamespaces { + if config.Config.Tap.AllNamespaces { return []string{mizu.K8sAllNamespaces} - } else if len(mizu.Config.Tap.Namespaces) > 0 { - return mizu.Config.Tap.Namespaces + } else if len(config.Config.Tap.Namespaces) > 0 { + return config.Config.Tap.Namespaces } else { return []string{kubernetesProvider.CurrentNamespace()} } diff --git a/cli/cmd/version.go b/cli/cmd/version.go index 45fc6dc9a..816f92865 100644 --- a/cli/cmd/version.go +++ b/cli/cmd/version.go @@ -1,27 +1,31 @@ package cmd import ( + "github.com/up9inc/mizu/cli/config" + "github.com/up9inc/mizu/cli/config/configStructs" + "github.com/up9inc/mizu/cli/logger" + "github.com/up9inc/mizu/cli/telemetry" "strconv" "time" "github.com/creasty/defaults" "github.com/spf13/cobra" "github.com/up9inc/mizu/cli/mizu" - "github.com/up9inc/mizu/cli/mizu/configStructs" ) var versionCmd = &cobra.Command{ Use: "version", Short: "Print version info", RunE: func(cmd *cobra.Command, args []string) error { - go mizu.ReportRun("version", mizu.Config.Version) - if mizu.Config.Version.DebugInfo { + go telemetry.ReportRun("version", config.Config.Version) + + if config.Config.Version.DebugInfo { timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0) - mizu.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash) - mizu.Log.Infof("Build Time: %s (%s)", mizu.BuildTimestamp, time.Unix(timeStampInt, 0)) + logger.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash) + logger.Log.Infof("Build Time: %s (%s)", mizu.BuildTimestamp, time.Unix(timeStampInt, 0)) } else { - mizu.Log.Infof("Version: %s (%s)", mizu.SemVer, mizu.Branch) + logger.Log.Infof("Version: %s (%s)", mizu.SemVer, mizu.Branch) } return nil }, diff --git a/cli/cmd/view.go b/cli/cmd/view.go index 8f9742e2a..a8379cf42 100644 --- a/cli/cmd/view.go +++ b/cli/cmd/view.go @@ -3,15 +3,16 @@ package cmd import ( "github.com/creasty/defaults" "github.com/spf13/cobra" - "github.com/up9inc/mizu/cli/mizu" - "github.com/up9inc/mizu/cli/mizu/configStructs" + "github.com/up9inc/mizu/cli/config" + "github.com/up9inc/mizu/cli/config/configStructs" + "github.com/up9inc/mizu/cli/telemetry" ) var viewCmd = &cobra.Command{ Use: "view", Short: "Open GUI in browser", RunE: func(cmd *cobra.Command, args []string) error { - go mizu.ReportRun("view", mizu.Config.View) + go telemetry.ReportRun("view", config.Config.View) runMizuView() return nil }, @@ -24,5 +25,4 @@ func init() { defaults.Set(&defaultViewConfig) viewCmd.Flags().Uint16P(configStructs.GuiPortViewName, "p", defaultViewConfig.GuiPort, "Provide a custom port for the web interface webserver") - viewCmd.Flags().StringP(configStructs.KubeConfigPathViewName, "k", defaultViewConfig.KubeConfigPath, "Path to kube-config file") } diff --git a/cli/cmd/viewRunner.go b/cli/cmd/viewRunner.go index 4b2b11a0d..11db07187 100644 --- a/cli/cmd/viewRunner.go +++ b/cli/cmd/viewRunner.go @@ -3,46 +3,51 @@ package cmd import ( "context" "fmt" + "github.com/up9inc/mizu/cli/config" "github.com/up9inc/mizu/cli/kubernetes" + "github.com/up9inc/mizu/cli/logger" "github.com/up9inc/mizu/cli/mizu" + "github.com/up9inc/mizu/cli/mizu/version" "net/http" + "time" ) func runMizuView() { - kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.View.KubeConfigPath) + kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath()) if err != nil { - mizu.Log.Error(err) + logger.Log.Error(err) return } ctx, cancel := context.WithCancel(context.Background()) defer cancel() - exists, err := kubernetesProvider.DoesServicesExist(ctx, mizu.Config.MizuResourcesNamespace, mizu.ApiServerPodName) + exists, err := kubernetesProvider.DoesServicesExist(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName) if err != nil { - mizu.Log.Errorf("Failed to found mizu service %v", err) + logger.Log.Errorf("Failed to found mizu service %v", err) cancel() return } if !exists { - mizu.Log.Infof("%s service not found, you should run `mizu tap` command first", mizu.ApiServerPodName) + logger.Log.Infof("%s service not found, you should run `mizu tap` command first", mizu.ApiServerPodName) cancel() return } - mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.View.GuiPort) + mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.View.GuiPort) _, err = http.Get(fmt.Sprintf("http://%s/", mizuProxiedUrl)) if err == nil { - mizu.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, mizu.Config.View.GuiPort) + logger.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, config.Config.View.GuiPort) return } - mizu.Log.Debugf("Found service %s, creating k8s proxy", mizu.ApiServerPodName) - + logger.Log.Infof("Establishing connection to k8s cluster...") go startProxyReportErrorIfAny(kubernetesProvider, cancel) - mizu.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.View.GuiPort)) - if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.View.GuiPort); err != nil { - mizu.Log.Errorf("Failed to check versions compatibility %v", err) + time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready + + logger.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.View.GuiPort)) + if isCompatible, err := version.CheckVersionCompatibility(config.Config.View.GuiPort); err != nil { + logger.Log.Errorf("Failed to check versions compatibility %v", err) cancel() return } else if !isCompatible { diff --git a/cli/config/config.go b/cli/config/config.go new file mode 100644 index 000000000..e1f66c036 --- /dev/null +++ b/cli/config/config.go @@ -0,0 +1,325 @@ +package config + +import ( + "errors" + "fmt" + "github.com/up9inc/mizu/cli/logger" + "github.com/up9inc/mizu/cli/mizu" + "io/ioutil" + "os" + "path" + "reflect" + "strconv" + "strings" + + "github.com/creasty/defaults" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/up9inc/mizu/cli/uiUtils" + "gopkg.in/yaml.v3" +) + +const ( + Separator = "=" + SetCommandName = "set" + FieldNameTag = "yaml" + ReadonlyTag = "readonly" +) + +var ( + Config = ConfigStruct{} + cmdName string +) + +func InitConfig(cmd *cobra.Command) error { + cmdName = cmd.Name() + + if err := defaults.Set(&Config); err != nil { + return err + } + + if err := mergeConfigFile(); err != nil { + return fmt.Errorf("invalid config, %w\n" + + "you can regenerate the file by removing it (%v) and using `mizu config -r`", err, GetConfigFilePath()) + } + + cmd.Flags().Visit(initFlag) + + finalConfigPrettified, _ := uiUtils.PrettyJson(Config) + logger.Log.Debugf("Init config finished\n Final config: %v", finalConfigPrettified) + + return nil +} + +func GetConfigWithDefaults() (string, error) { + defaultConf := ConfigStruct{} + if err := defaults.Set(&defaultConf); err != nil { + return "", err + } + + configElem := reflect.ValueOf(&defaultConf).Elem() + setZeroForReadonlyFields(configElem) + + return uiUtils.PrettyYaml(defaultConf) +} + +func GetConfigFilePath() string { + return path.Join(mizu.GetMizuFolderPath(), "config.yaml") +} + +func mergeConfigFile() error { + reader, openErr := os.Open(GetConfigFilePath()) + if openErr != nil { + return nil + } + + buf, readErr := ioutil.ReadAll(reader) + if readErr != nil { + return readErr + } + + if err := yaml.Unmarshal(buf, &Config); err != nil { + return err + } + logger.Log.Debugf("Found config file, merged to default options") + + return nil +} + +func initFlag(f *pflag.Flag) { + configElemValue := reflect.ValueOf(&Config).Elem() + + flagPath := []string {cmdName, f.Name} + + sliceValue, isSliceValue := f.Value.(pflag.SliceValue) + if !isSliceValue { + if err := mergeFlagValue(configElemValue, flagPath, strings.Join(flagPath, "."), f.Value.String()); err != nil { + logger.Log.Warningf(uiUtils.Warning, err) + } + return + } + + if f.Name == SetCommandName { + if err := mergeSetFlag(configElemValue, sliceValue.GetSlice()); err != nil { + logger.Log.Warningf(uiUtils.Warning, err) + } + return + } + + if err := mergeFlagValues(configElemValue, flagPath, strings.Join(flagPath, "."), sliceValue.GetSlice()); err != nil { + logger.Log.Warningf(uiUtils.Warning, err) + } +} + +func mergeSetFlag(configElemValue reflect.Value, setValues []string) error { + var setErrors []string + setMap := map[string][]string{} + + for _, setValue := range setValues { + if !strings.Contains(setValue, Separator) { + setErrors = append(setErrors, fmt.Sprintf("Ignoring set argument %s (set argument format: =)", setValue)) + continue + } + + split := strings.SplitN(setValue, Separator, 2) + argumentKey, argumentValue := split[0], split[1] + + setMap[argumentKey] = append(setMap[argumentKey], argumentValue) + } + + for argumentKey, argumentValues := range setMap { + flagPath := strings.Split(argumentKey, ".") + + if len(argumentValues) > 1 { + if err := mergeFlagValues(configElemValue, flagPath, argumentKey, argumentValues); err != nil { + setErrors = append(setErrors, fmt.Sprintf("%v", err)) + } + } else { + if err := mergeFlagValue(configElemValue, flagPath, argumentKey, argumentValues[0]); err != nil { + setErrors = append(setErrors, fmt.Sprintf("%v", err)) + } + } + } + + if len(setErrors) > 0 { + return fmt.Errorf(strings.Join(setErrors, "\n")) + } + + return nil +} + +func mergeFlagValue(configElemValue reflect.Value, flagPath []string, fullFlagName string, flagValue string) error { + mergeFunction := func(flagName string, currentFieldStruct reflect.StructField, currentFieldElemValue reflect.Value, currentElemValue reflect.Value) error { + currentFieldKind := currentFieldStruct.Type.Kind() + + if currentFieldKind == reflect.Slice { + return mergeFlagValues(currentElemValue, []string{flagName}, fullFlagName, []string{flagValue}) + } + + parsedValue, err := getParsedValue(currentFieldKind, flagValue) + if err != nil { + return fmt.Errorf("invalid value %s for flag name %s, expected %s", flagValue, flagName, currentFieldKind) + } + + currentFieldElemValue.Set(parsedValue) + return nil + } + + return mergeFlag(configElemValue, flagPath, fullFlagName, mergeFunction) +} + +func mergeFlagValues(configElemValue reflect.Value, flagPath []string, fullFlagName string, flagValues []string) error { + mergeFunction := func(flagName string, currentFieldStruct reflect.StructField, currentFieldElemValue reflect.Value, currentElemValue reflect.Value) error { + currentFieldKind := currentFieldStruct.Type.Kind() + + if currentFieldKind != reflect.Slice { + return fmt.Errorf("invalid values %s for flag name %s, expected %s", strings.Join(flagValues, ","), flagName, currentFieldKind) + } + + flagValueKind := currentFieldStruct.Type.Elem().Kind() + + parsedValues := reflect.MakeSlice(reflect.SliceOf(currentFieldStruct.Type.Elem()), 0, 0) + for _, flagValue := range flagValues { + parsedValue, err := getParsedValue(flagValueKind, flagValue) + if err != nil { + return fmt.Errorf("invalid value %s for flag name %s, expected %s", flagValue, flagName, flagValueKind) + } + + parsedValues = reflect.Append(parsedValues, parsedValue) + } + + currentFieldElemValue.Set(parsedValues) + return nil + } + + return mergeFlag(configElemValue, flagPath, fullFlagName, mergeFunction) +} + +func mergeFlag(currentElemValue reflect.Value, currentFlagPath []string, fullFlagName string, mergeFunction func(flagName string, currentFieldStruct reflect.StructField, currentFieldElemValue reflect.Value, currentElemValue reflect.Value) error) error { + if len(currentFlagPath) == 0 { + return fmt.Errorf("flag \"%s\" not found", fullFlagName) + } + + for i := 0; i < currentElemValue.NumField(); i++ { + currentFieldStruct := currentElemValue.Type().Field(i) + currentFieldElemValue := currentElemValue.FieldByName(currentFieldStruct.Name) + + if currentFieldStruct.Type.Kind() == reflect.Struct && getFieldNameByTag(currentFieldStruct) == currentFlagPath[0] { + return mergeFlag(currentFieldElemValue, currentFlagPath[1:], fullFlagName, mergeFunction) + } + + if len(currentFlagPath) > 1 || getFieldNameByTag(currentFieldStruct) != currentFlagPath[0] { + continue + } + + return mergeFunction(currentFlagPath[0], currentFieldStruct, currentFieldElemValue, currentElemValue) + } + + return fmt.Errorf("flag \"%s\" not found", fullFlagName) +} + +func getFieldNameByTag(field reflect.StructField) string { + return strings.Split(field.Tag.Get(FieldNameTag), ",")[0] +} + +func getParsedValue(kind reflect.Kind, value string) (reflect.Value, error) { + switch kind { + case reflect.String: + return reflect.ValueOf(value), nil + case reflect.Bool: + boolArgumentValue, err := strconv.ParseBool(value) + if err != nil { + break + } + + return reflect.ValueOf(boolArgumentValue), nil + case reflect.Int: + intArgumentValue, err := strconv.ParseInt(value, 10, 64) + if err != nil { + break + } + + return reflect.ValueOf(int(intArgumentValue)), nil + case reflect.Int8: + intArgumentValue, err := strconv.ParseInt(value, 10, 8) + if err != nil { + break + } + + return reflect.ValueOf(int8(intArgumentValue)), nil + case reflect.Int16: + intArgumentValue, err := strconv.ParseInt(value, 10, 16) + if err != nil { + break + } + + return reflect.ValueOf(int16(intArgumentValue)), nil + case reflect.Int32: + intArgumentValue, err := strconv.ParseInt(value, 10, 32) + if err != nil { + break + } + + return reflect.ValueOf(int32(intArgumentValue)), nil + case reflect.Int64: + intArgumentValue, err := strconv.ParseInt(value, 10, 64) + if err != nil { + break + } + + return reflect.ValueOf(intArgumentValue), nil + case reflect.Uint: + uintArgumentValue, err := strconv.ParseUint(value, 10, 64) + if err != nil { + break + } + + return reflect.ValueOf(uint(uintArgumentValue)), nil + case reflect.Uint8: + uintArgumentValue, err := strconv.ParseUint(value, 10, 8) + if err != nil { + break + } + + return reflect.ValueOf(uint8(uintArgumentValue)), nil + case reflect.Uint16: + uintArgumentValue, err := strconv.ParseUint(value, 10, 16) + if err != nil { + break + } + + return reflect.ValueOf(uint16(uintArgumentValue)), nil + case reflect.Uint32: + uintArgumentValue, err := strconv.ParseUint(value, 10, 32) + if err != nil { + break + } + + return reflect.ValueOf(uint32(uintArgumentValue)), nil + case reflect.Uint64: + uintArgumentValue, err := strconv.ParseUint(value, 10, 64) + if err != nil { + break + } + + return reflect.ValueOf(uintArgumentValue), nil + } + + return reflect.ValueOf(nil), errors.New("value to parse does not match type") +} + +func setZeroForReadonlyFields(currentElem reflect.Value) { + for i := 0; i < currentElem.NumField(); i++ { + currentField := currentElem.Type().Field(i) + currentFieldByName := currentElem.FieldByName(currentField.Name) + + if currentField.Type.Kind() == reflect.Struct { + setZeroForReadonlyFields(currentFieldByName) + continue + } + + if _, ok := currentField.Tag.Lookup(ReadonlyTag); ok { + currentFieldByName.Set(reflect.Zero(currentField.Type)) + } + } +} diff --git a/cli/mizu/configStruct.go b/cli/config/configStruct.go similarity index 50% rename from cli/mizu/configStruct.go rename to cli/config/configStruct.go index cd7d9bad3..aa4de7944 100644 --- a/cli/mizu/configStruct.go +++ b/cli/config/configStruct.go @@ -1,17 +1,17 @@ -package mizu +package config import ( "fmt" - - "github.com/up9inc/mizu/cli/mizu/configStructs" + "github.com/up9inc/mizu/cli/config/configStructs" + "github.com/up9inc/mizu/cli/mizu" + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/util/homedir" + "os" + "path/filepath" ) const ( - AgentImageConfigName = "agent-image" MizuResourcesNamespaceConfigName = "mizu-resources-namespace" - TelemetryConfigName = "telemetry" - DumpLogsConfigName = "dump-logs" - KubeConfigPathName = "kube-config-path" ) type ConfigStruct struct { @@ -19,17 +19,38 @@ type ConfigStruct struct { Fetch configStructs.FetchConfig `yaml:"fetch"` Version configStructs.VersionConfig `yaml:"version"` View configStructs.ViewConfig `yaml:"view"` - AgentImage string `yaml:"agent-image,omitempty"` + Logs configStructs.LogsConfig `yaml:"logs"` + Config configStructs.ConfigConfig `yaml:"config,omitempty"` + AgentImage string `yaml:"agent-image,omitempty" readonly:""` + ImagePullPolicyStr string `yaml:"image-pull-policy" default:"Always"` MizuResourcesNamespace string `yaml:"mizu-resources-namespace" default:"mizu"` Telemetry bool `yaml:"telemetry" default:"true"` DumpLogs bool `yaml:"dump-logs" default:"false"` - KubeConfigPath string `yaml:"kube-config-path" default:""` + KubeConfigPathStr string `yaml:"kube-config-path"` } func (config *ConfigStruct) SetDefaults() { - config.AgentImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", Branch, SemVer) + config.AgentImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", mizu.Branch, mizu.SemVer) +} + +func (config *ConfigStruct) ImagePullPolicy() v1.PullPolicy { + return v1.PullPolicy(config.ImagePullPolicyStr) } func (config *ConfigStruct) IsNsRestrictedMode() bool { return config.MizuResourcesNamespace != "mizu" // Notice "mizu" string must match the default MizuResourcesNamespace } + +func (config *ConfigStruct) KubeConfigPath() string { + if config.KubeConfigPathStr != "" { + return config.KubeConfigPathStr + } + + envKubeConfigPath := os.Getenv("KUBECONFIG") + if envKubeConfigPath != "" { + return envKubeConfigPath + } + + home := homedir.HomeDir() + return filepath.Join(home, ".kube", "config") +} diff --git a/cli/config/configStructs/configConfig.go b/cli/config/configStructs/configConfig.go new file mode 100644 index 000000000..3fc9a5626 --- /dev/null +++ b/cli/config/configStructs/configConfig.go @@ -0,0 +1,9 @@ +package configStructs + +const ( + RegenerateConfigName = "regenerate" +) + +type ConfigConfig struct { + Regenerate bool `yaml:"regenerate,omitempty" default:"false" readonly:""` +} diff --git a/cli/mizu/configStructs/fetchConfig.go b/cli/config/configStructs/fetchConfig.go similarity index 100% rename from cli/mizu/configStructs/fetchConfig.go rename to cli/config/configStructs/fetchConfig.go diff --git a/cli/config/configStructs/logsConfig.go b/cli/config/configStructs/logsConfig.go new file mode 100644 index 000000000..90d41888f --- /dev/null +++ b/cli/config/configStructs/logsConfig.go @@ -0,0 +1,35 @@ +package configStructs + +import ( + "fmt" + "os" + "path" +) + +const ( + FileLogsName = "file" +) + +type LogsConfig struct { + FileStr string `yaml:"file"` +} + +func (config *LogsConfig) Validate() error { + if config.FileStr == "" { + _, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get PWD, %v (try using `mizu logs -f )`", err) + } + } + + return nil +} + +func (config *LogsConfig) FilePath() string { + if config.FileStr == "" { + pwd, _ := os.Getwd() + return path.Join(pwd, "mizu_logs.zip") + } + + return config.FileStr +} \ No newline at end of file diff --git a/cli/mizu/configStructs/tapConfig.go b/cli/config/configStructs/tapConfig.go similarity index 57% rename from cli/mizu/configStructs/tapConfig.go rename to cli/config/configStructs/tapConfig.go index 67582e467..d80163b27 100644 --- a/cli/mizu/configStructs/tapConfig.go +++ b/cli/config/configStructs/tapConfig.go @@ -10,14 +10,11 @@ import ( ) const ( - AnalysisDestinationTapName = "dest" - SleepIntervalSecTapName = "upload-interval" GuiPortTapName = "gui-port" NamespacesTapName = "namespaces" AnalysisTapName = "analysis" AllNamespacesTapName = "all-namespaces" PlainTextFilterRegexesTapName = "regex-masking" - HideHealthChecksTapName = "hide-healthchecks" DisableRedactionTapName = "no-redact" HumanMaxEntriesDBSizeTapName = "max-entries-db-size" DirectionTapName = "direction" @@ -26,20 +23,29 @@ const ( ) type TapConfig struct { - AnalysisDestination string `yaml:"dest" default:"up9.app"` - SleepIntervalSec int `yaml:"upload-interval" default:"10"` - PodRegexStr string `yaml:"regex" default:".*"` - GuiPort uint16 `yaml:"gui-port" default:"8899"` - Namespaces []string `yaml:"namespaces"` - Analysis bool `yaml:"analysis" default:"false"` - AllNamespaces bool `yaml:"all-namespaces" default:"false"` - PlainTextFilterRegexes []string `yaml:"regex-masking"` - HideHealthChecks bool `yaml:"hide-healthchecks" default:"false"` - DisableRedaction bool `yaml:"no-redact" default:"false"` - HumanMaxEntriesDBSize string `yaml:"max-entries-db-size" default:"200MB"` - Direction string `yaml:"direction" default:"in"` - DryRun bool `yaml:"dry-run" default:"false"` - EnforcePolicyFile string `yaml:"test-rules"` + AnalysisDestination string `yaml:"dest" default:"up9.app"` + SleepIntervalSec int `yaml:"upload-interval" default:"10"` + PodRegexStr string `yaml:"regex" default:".*"` + GuiPort uint16 `yaml:"gui-port" default:"8899"` + Namespaces []string `yaml:"namespaces"` + Analysis bool `yaml:"analysis" default:"false"` + AllNamespaces bool `yaml:"all-namespaces" default:"false"` + PlainTextFilterRegexes []string `yaml:"regex-masking"` + HealthChecksUserAgentHeaders []string `yaml:"ignored-user-agents"` + DisableRedaction bool `yaml:"no-redact" default:"false"` + HumanMaxEntriesDBSize string `yaml:"max-entries-db-size" default:"200MB"` + Direction string `yaml:"direction" default:"in"` + DryRun bool `yaml:"dry-run" default:"false"` + EnforcePolicyFile string `yaml:"test-rules"` + ApiServerResources Resources `yaml:"api-server-resources"` + TapperResources Resources `yaml:"tapper-resources"` +} + +type Resources struct { + CpuLimit string `yaml:"cpu-limit" default:"750m"` + MemoryLimit string `yaml:"memory-limit" default:"1Gi"` + CpuRequests string `yaml:"cpu-requests" default:"50m"` + MemoryRequests string `yaml:"memory-requests" default:"50Mi"` } func (config *TapConfig) PodRegex() *regexp.Regexp { diff --git a/cli/mizu/configStructs/versionConfig.go b/cli/config/configStructs/versionConfig.go similarity index 100% rename from cli/mizu/configStructs/versionConfig.go rename to cli/config/configStructs/versionConfig.go diff --git a/cli/config/configStructs/viewConfig.go b/cli/config/configStructs/viewConfig.go new file mode 100644 index 000000000..aa41a7353 --- /dev/null +++ b/cli/config/configStructs/viewConfig.go @@ -0,0 +1,9 @@ +package configStructs + +const ( + GuiPortViewName = "gui-port" +) + +type ViewConfig struct { + GuiPort uint16 `yaml:"gui-port" default:"8899"` +} diff --git a/cli/config/config_internal_test.go b/cli/config/config_internal_test.go new file mode 100644 index 000000000..95789be32 --- /dev/null +++ b/cli/config/config_internal_test.go @@ -0,0 +1,385 @@ +package config + +import ( + "fmt" + "reflect" + "testing" +) + +type ConfigMock struct { + SectionMock SectionMock `yaml:"section"` + Test string `yaml:"test"` + StringField string `yaml:"string-field"` + IntField int `yaml:"int-field"` + BoolField bool `yaml:"bool-field"` + UintField uint `yaml:"uint-field"` + StringSliceField []string `yaml:"string-slice-field"` + IntSliceField []int `yaml:"int-slice-field"` + BoolSliceField []bool `yaml:"bool-slice-field"` + UintSliceField []uint `yaml:"uint-slice-field"` +} + +type SectionMock struct { + Test string `yaml:"test"` +} + +type FieldSetValues struct { + SetValues []string + FieldName string + FieldValue interface{} +} + +func TestMergeSetFlagNoSeparator(t *testing.T) { + tests := []struct { + Name string + SetValues []string + }{ + {Name: "empty value", SetValues: []string{""}}, + {Name: "single char", SetValues: []string{"t"}}, + {Name: "combine empty value and single char", SetValues: []string{"", "t"}}, + {Name: "two values without separator", SetValues: []string{"test", "test:true"}}, + {Name: "four values without separator", SetValues: []string{"test", "test:true", "testing!", "true"}}, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + configMock := ConfigMock{} + configMockElemValue := reflect.ValueOf(&configMock).Elem() + + err := mergeSetFlag(configMockElemValue, test.SetValues) + + if err == nil { + t.Errorf("unexpected unhandled error - SetValues: %v", test.SetValues) + return + } + + for i := 0; i < configMockElemValue.NumField(); i++ { + currentField := configMockElemValue.Type().Field(i) + currentFieldByName := configMockElemValue.FieldByName(currentField.Name) + + if !currentFieldByName.IsZero() { + t.Errorf("unexpected value with not default value - SetValues: %v", test.SetValues) + } + } + }) + } +} + +func TestMergeSetFlagInvalidFlagName(t *testing.T) { + tests := []struct { + Name string + SetValues []string + }{ + {Name: "invalid flag name", SetValues: []string{"invalid_flag=true"}}, + {Name: "invalid flag name inside section struct", SetValues: []string{"section.invalid_flag=test"}}, + {Name: "flag name is a struct", SetValues: []string{"section=test"}}, + {Name: "empty flag name", SetValues: []string{"=true"}}, + {Name: "four tests combined", SetValues: []string{"invalid_flag=true", "config.invalid_flag=test", "section=test", "=true"}}, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + configMock := ConfigMock{} + configMockElemValue := reflect.ValueOf(&configMock).Elem() + + err := mergeSetFlag(configMockElemValue, test.SetValues) + + if err == nil { + t.Errorf("unexpected unhandled error - SetValues: %v", test.SetValues) + return + } + + for i := 0; i < configMockElemValue.NumField(); i++ { + currentField := configMockElemValue.Type().Field(i) + currentFieldByName := configMockElemValue.FieldByName(currentField.Name) + + if !currentFieldByName.IsZero() { + t.Errorf("unexpected case - SetValues: %v", test.SetValues) + } + } + }) + } +} + +func TestMergeSetFlagInvalidFlagValue(t *testing.T) { + tests := []struct { + Name string + SetValues []string + }{ + {Name: "bool value to int field", SetValues: []string{"int-field=true"}}, + {Name: "int value to bool field", SetValues: []string{"bool-field:5"}}, + {Name: "int value to uint field", SetValues: []string{"uint-field=-1"}}, + {Name: "bool value to int slice field", SetValues: []string{"int-slice-field=true"}}, + {Name: "int value to bool slice field", SetValues: []string{"bool-slice-field=5"}}, + {Name: "int value to uint slice field", SetValues: []string{"uint-slice-field=-1"}}, + {Name: "int slice value to int field", SetValues: []string{"int-field=6", "int-field=66"}}, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + configMock := ConfigMock{} + configMockElemValue := reflect.ValueOf(&configMock).Elem() + + err := mergeSetFlag(configMockElemValue, test.SetValues) + + if err == nil { + t.Errorf("unexpected unhandled error - SetValues: %v", test.SetValues) + return + } + + for i := 0; i < configMockElemValue.NumField(); i++ { + currentField := configMockElemValue.Type().Field(i) + currentFieldByName := configMockElemValue.FieldByName(currentField.Name) + + if !currentFieldByName.IsZero() { + t.Errorf("unexpected case - SetValues: %v", test.SetValues) + } + } + }) + } +} + +func TestMergeSetFlagNotSliceValues(t *testing.T) { + tests := []struct { + Name string + FieldsSetValues []FieldSetValues + }{ + {Name: "string field", FieldsSetValues: []FieldSetValues{{SetValues: []string{"string-field=test"}, FieldName: "StringField", FieldValue: "test"}}}, + {Name: "int field", FieldsSetValues: []FieldSetValues{{SetValues: []string{"int-field=6"}, FieldName: "IntField", FieldValue: 6}}}, + {Name: "bool field", FieldsSetValues: []FieldSetValues{{SetValues: []string{"bool-field=true"}, FieldName: "BoolField", FieldValue: true}}}, + {Name: "uint field", FieldsSetValues: []FieldSetValues{{SetValues: []string{"uint-field=6"}, FieldName: "UintField", FieldValue: uint(6)}}}, + {Name: "four fields combined", FieldsSetValues: []FieldSetValues { + {SetValues: []string{"string-field=test"}, FieldName: "StringField", FieldValue: "test"}, + {SetValues: []string{"int-field=6"}, FieldName: "IntField", FieldValue: 6}, + {SetValues: []string{"bool-field=true"}, FieldName: "BoolField", FieldValue: true}, + {SetValues: []string{"uint-field=6"}, FieldName: "UintField", FieldValue: uint(6)}, + }}, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + configMock := ConfigMock{} + configMockElemValue := reflect.ValueOf(&configMock).Elem() + + var setValues []string + for _, fieldSetValues := range test.FieldsSetValues { + setValues = append(setValues, fieldSetValues.SetValues...) + } + + err := mergeSetFlag(configMockElemValue, setValues) + + if err != nil { + t.Errorf("unexpected error result - err: %v", err) + return + } + + for _, fieldSetValues := range test.FieldsSetValues { + fieldValue := configMockElemValue.FieldByName(fieldSetValues.FieldName).Interface() + if fieldValue != fieldSetValues.FieldValue { + t.Errorf("unexpected result - expected: %v, actual: %v", fieldSetValues.FieldValue, fieldValue) + } + } + }) + } +} + +func TestMergeSetFlagSliceValues(t *testing.T) { + tests := []struct { + Name string + FieldsSetValues []FieldSetValues + }{ + {Name: "string slice field single value", FieldsSetValues: []FieldSetValues{{SetValues: []string{"string-slice-field=test"}, FieldName: "StringSliceField", FieldValue: []string{"test"}}}}, + {Name: "int slice field single value", FieldsSetValues: []FieldSetValues{{SetValues: []string{"int-slice-field=6"}, FieldName: "IntSliceField", FieldValue: []int{6}}}}, + {Name: "bool slice field single value", FieldsSetValues: []FieldSetValues{{SetValues: []string{"bool-slice-field=true"}, FieldName: "BoolSliceField", FieldValue: []bool{true}}}}, + {Name: "uint slice field single value", FieldsSetValues: []FieldSetValues{{SetValues: []string{"uint-slice-field=6"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6)}}}}, + {Name: "four single value fields combined", FieldsSetValues: []FieldSetValues{ + {SetValues: []string{"string-slice-field=test"}, FieldName: "StringSliceField", FieldValue: []string{"test"}}, + {SetValues: []string{"int-slice-field=6"}, FieldName: "IntSliceField", FieldValue: []int{6}}, + {SetValues: []string{"bool-slice-field=true"}, FieldName: "BoolSliceField", FieldValue: []bool{true}}, + {SetValues: []string{"uint-slice-field=6"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6)}}, + }}, + {Name: "string slice field two values", FieldsSetValues: []FieldSetValues{{SetValues: []string{"string-slice-field=test", "string-slice-field=test2"}, FieldName: "StringSliceField", FieldValue: []string{"test", "test2"}}}}, + {Name: "int slice field two values", FieldsSetValues: []FieldSetValues{{SetValues: []string{"int-slice-field=6", "int-slice-field=66"}, FieldName: "IntSliceField", FieldValue: []int{6, 66}}}}, + {Name: "bool slice field two values", FieldsSetValues: []FieldSetValues{{SetValues: []string{"bool-slice-field=true", "bool-slice-field=false"}, FieldName: "BoolSliceField", FieldValue: []bool{true, false}}}}, + {Name: "uint slice field two values", FieldsSetValues: []FieldSetValues{{SetValues: []string{"uint-slice-field=6", "uint-slice-field=66"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6), uint(66)}}}}, + {Name: "four two values fields combined", FieldsSetValues: []FieldSetValues{ + {SetValues: []string{"string-slice-field=test", "string-slice-field=test2"}, FieldName: "StringSliceField", FieldValue: []string{"test", "test2"}}, + {SetValues: []string{"int-slice-field=6", "int-slice-field=66"}, FieldName: "IntSliceField", FieldValue: []int{6, 66}}, + {SetValues: []string{"bool-slice-field=true", "bool-slice-field=false"}, FieldName: "BoolSliceField", FieldValue: []bool{true, false}}, + {SetValues: []string{"uint-slice-field=6", "uint-slice-field=66"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6), uint(66)}}, + }}, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + configMock := ConfigMock{} + configMockElemValue := reflect.ValueOf(&configMock).Elem() + + var setValues []string + for _, fieldSetValues := range test.FieldsSetValues { + setValues = append(setValues, fieldSetValues.SetValues...) + } + + err := mergeSetFlag(configMockElemValue, setValues) + + if err != nil { + t.Errorf("unexpected error result - err: %v", err) + return + } + + for _, fieldSetValues := range test.FieldsSetValues { + fieldValue := configMockElemValue.FieldByName(fieldSetValues.FieldName).Interface() + if !reflect.DeepEqual(fieldValue, fieldSetValues.FieldValue) { + t.Errorf("unexpected result - expected: %v, actual: %v", fieldSetValues.FieldValue, fieldValue) + } + } + }) + } +} + +func TestMergeSetFlagMixValues(t *testing.T) { + tests := []struct { + Name string + FieldsSetValues []FieldSetValues + }{ + {Name: "single value all fields", FieldsSetValues: []FieldSetValues{ + {SetValues: []string{"string-slice-field=test"}, FieldName: "StringSliceField", FieldValue: []string{"test"}}, + {SetValues: []string{"int-slice-field=6"}, FieldName: "IntSliceField", FieldValue: []int{6}}, + {SetValues: []string{"bool-slice-field=true"}, FieldName: "BoolSliceField", FieldValue: []bool{true}}, + {SetValues: []string{"uint-slice-field=6"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6)}}, + {SetValues: []string{"string-field=test"}, FieldName: "StringField", FieldValue: "test"}, + {SetValues: []string{"int-field=6"}, FieldName: "IntField", FieldValue: 6}, + {SetValues: []string{"bool-field=true"}, FieldName: "BoolField", FieldValue: true}, + {SetValues: []string{"uint-field=6"}, FieldName: "UintField", FieldValue: uint(6)}, + }}, + {Name: "two values slice fields and single value fields", FieldsSetValues: []FieldSetValues{ + {SetValues: []string{"string-slice-field=test", "string-slice-field=test2"}, FieldName: "StringSliceField", FieldValue: []string{"test", "test2"}}, + {SetValues: []string{"int-slice-field=6", "int-slice-field=66"}, FieldName: "IntSliceField", FieldValue: []int{6, 66}}, + {SetValues: []string{"bool-slice-field=true", "bool-slice-field=false"}, FieldName: "BoolSliceField", FieldValue: []bool{true, false}}, + {SetValues: []string{"uint-slice-field=6", "uint-slice-field=66"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6), uint(66)}}, + {SetValues: []string{"string-field=test"}, FieldName: "StringField", FieldValue: "test"}, + {SetValues: []string{"int-field=6"}, FieldName: "IntField", FieldValue: 6}, + {SetValues: []string{"bool-field=true"}, FieldName: "BoolField", FieldValue: true}, + {SetValues: []string{"uint-field=6"}, FieldName: "UintField", FieldValue: uint(6)}, + }}, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + configMock := ConfigMock{} + configMockElemValue := reflect.ValueOf(&configMock).Elem() + + var setValues []string + for _, fieldSetValues := range test.FieldsSetValues { + setValues = append(setValues, fieldSetValues.SetValues...) + } + + err := mergeSetFlag(configMockElemValue, setValues) + + if err != nil { + t.Errorf("unexpected error result - err: %v", err) + return + } + + for _, fieldSetValues := range test.FieldsSetValues { + fieldValue := configMockElemValue.FieldByName(fieldSetValues.FieldName).Interface() + if !reflect.DeepEqual(fieldValue, fieldSetValues.FieldValue) { + t.Errorf("unexpected result - expected: %v, actual: %v", fieldSetValues.FieldValue, fieldValue) + } + } + }) + } +} + +func TestGetParsedValueValidValue(t *testing.T) { + tests := []struct { + StringValue string + Kind reflect.Kind + ActualValue interface{} + }{ + {StringValue: "test", Kind: reflect.String, ActualValue: "test"}, + {StringValue: "123", Kind: reflect.String, ActualValue: "123"}, + {StringValue: "true", Kind: reflect.Bool, ActualValue: true}, + {StringValue: "false", Kind: reflect.Bool, ActualValue: false}, + {StringValue: "6", Kind: reflect.Int, ActualValue: 6}, + {StringValue: "-6", Kind: reflect.Int, ActualValue: -6}, + {StringValue: "6", Kind: reflect.Int8, ActualValue: int8(6)}, + {StringValue: "-6", Kind: reflect.Int8, ActualValue: int8(-6)}, + {StringValue: "6", Kind: reflect.Int16, ActualValue: int16(6)}, + {StringValue: "-6", Kind: reflect.Int16, ActualValue: int16(-6)}, + {StringValue: "6", Kind: reflect.Int32, ActualValue: int32(6)}, + {StringValue: "-6", Kind: reflect.Int32, ActualValue: int32(-6)}, + {StringValue: "6", Kind: reflect.Int64, ActualValue: int64(6)}, + {StringValue: "-6", Kind: reflect.Int64, ActualValue: int64(-6)}, + {StringValue: "6", Kind: reflect.Uint, ActualValue: uint(6)}, + {StringValue: "66", Kind: reflect.Uint, ActualValue: uint(66)}, + {StringValue: "6", Kind: reflect.Uint8, ActualValue: uint8(6)}, + {StringValue: "66", Kind: reflect.Uint8, ActualValue: uint8(66)}, + {StringValue: "6", Kind: reflect.Uint16, ActualValue: uint16(6)}, + {StringValue: "66", Kind: reflect.Uint16, ActualValue: uint16(66)}, + {StringValue: "6", Kind: reflect.Uint32, ActualValue: uint32(6)}, + {StringValue: "66", Kind: reflect.Uint32, ActualValue: uint32(66)}, + {StringValue: "6", Kind: reflect.Uint64, ActualValue: uint64(6)}, + {StringValue: "66", Kind: reflect.Uint64, ActualValue: uint64(66)}, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%v %v", test.Kind, test.StringValue), func(t *testing.T) { + parsedValue, err := getParsedValue(test.Kind, test.StringValue) + + if err != nil { + t.Errorf("unexpected error result - err: %v", err) + return + } + + if parsedValue.Interface() != test.ActualValue { + t.Errorf("unexpected result - expected: %v, actual: %v", test.ActualValue, parsedValue) + } + }) + } +} + +func TestGetParsedValueInvalidValue(t *testing.T) { + tests := []struct { + StringValue string + Kind reflect.Kind + }{ + {StringValue: "test", Kind: reflect.Bool}, + {StringValue: "123", Kind: reflect.Bool}, + {StringValue: "test", Kind: reflect.Int}, + {StringValue: "true", Kind: reflect.Int}, + {StringValue: "test", Kind: reflect.Int8}, + {StringValue: "true", Kind: reflect.Int8}, + {StringValue: "test", Kind: reflect.Int16}, + {StringValue: "true", Kind: reflect.Int16}, + {StringValue: "test", Kind: reflect.Int32}, + {StringValue: "true", Kind: reflect.Int32}, + {StringValue: "test", Kind: reflect.Int64}, + {StringValue: "true", Kind: reflect.Int64}, + {StringValue: "test", Kind: reflect.Uint}, + {StringValue: "-6", Kind: reflect.Uint}, + {StringValue: "test", Kind: reflect.Uint8}, + {StringValue: "-6", Kind: reflect.Uint8}, + {StringValue: "test", Kind: reflect.Uint16}, + {StringValue: "-6", Kind: reflect.Uint16}, + {StringValue: "test", Kind: reflect.Uint32}, + {StringValue: "-6", Kind: reflect.Uint32}, + {StringValue: "test", Kind: reflect.Uint64}, + {StringValue: "-6", Kind: reflect.Uint64}, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%v %v", test.Kind, test.StringValue), func(t *testing.T) { + parsedValue, err := getParsedValue(test.Kind, test.StringValue) + + if err == nil { + t.Errorf("unexpected unhandled error - stringValue: %v, Kind: %v", test.StringValue, test.Kind) + return + } + + if parsedValue != reflect.ValueOf(nil) { + t.Errorf("unexpected parsed value - parsedValue: %v", parsedValue) + } + }) + } +} diff --git a/cli/config/config_test.go b/cli/config/config_test.go new file mode 100644 index 000000000..d5028cd94 --- /dev/null +++ b/cli/config/config_test.go @@ -0,0 +1,41 @@ +package config_test + +import ( + "github.com/up9inc/mizu/cli/config" + "reflect" + "strings" + "testing" +) + +func TestConfigWriteIgnoresReadonlyFields(t *testing.T) { + var readonlyFields []string + + configElem := reflect.ValueOf(&config.ConfigStruct{}).Elem() + getFieldsWithReadonlyTag(configElem, &readonlyFields) + + configWithDefaults, _ := config.GetConfigWithDefaults() + for _, readonlyField := range readonlyFields { + t.Run(readonlyField, func(t *testing.T) { + if strings.Contains(configWithDefaults, readonlyField) { + t.Errorf("unexpected result - readonly field: %v, config: %v", readonlyField, configWithDefaults) + } + }) + } +} + +func getFieldsWithReadonlyTag(currentElem reflect.Value, readonlyFields *[]string) { + for i := 0; i < currentElem.NumField(); i++ { + currentField := currentElem.Type().Field(i) + currentFieldByName := currentElem.FieldByName(currentField.Name) + + if currentField.Type.Kind() == reflect.Struct { + getFieldsWithReadonlyTag(currentFieldByName, readonlyFields) + continue + } + + if _, ok := currentField.Tag.Lookup(config.ReadonlyTag); ok { + fieldNameByTag := strings.Split(currentField.Tag.Get(config.FieldNameTag), ",")[0] + *readonlyFields = append(*readonlyFields, fieldNameByTag) + } + } +} diff --git a/cli/errormessage/errormessage.go b/cli/errormessage/errormessage.go index 1268f835d..7f9caac0d 100644 --- a/cli/errormessage/errormessage.go +++ b/cli/errormessage/errormessage.go @@ -3,9 +3,7 @@ package errormessage import ( "errors" "fmt" - - "github.com/up9inc/mizu/cli/mizu" - + "github.com/up9inc/mizu/cli/config" regexpsyntax "regexp/syntax" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -20,9 +18,9 @@ func FormatError(err error) error { "supply the required permission or control Mizu's access to namespaces by setting %s "+ "in the config file or setting the tapped namespace with --%s %s=", err, - mizu.MizuResourcesNamespaceConfigName, - mizu.SetCommandName, - mizu.MizuResourcesNamespaceConfigName) + config.MizuResourcesNamespaceConfigName, + config.SetCommandName, + config.MizuResourcesNamespaceConfigName) } else if syntaxError, isSyntaxError := asRegexSyntaxError(err); isSyntaxError { errorNew = fmt.Errorf("regex %s is invalid: %w", syntaxError.Expr, err) } else { diff --git a/cli/fsUtils/mizuLogsUtils.go b/cli/fsUtils/mizuLogsUtils.go deleted file mode 100644 index b48cc2aa6..000000000 --- a/cli/fsUtils/mizuLogsUtils.go +++ /dev/null @@ -1,58 +0,0 @@ -package fsUtils - -import ( - "archive/zip" - "context" - "fmt" - "github.com/up9inc/mizu/cli/kubernetes" - "github.com/up9inc/mizu/cli/mizu" - "os" - "regexp" -) - -func DumpLogs(provider *kubernetes.Provider, ctx context.Context, filePath string) error { - podExactRegex := regexp.MustCompile("^" + mizu.MizuResourcesPrefix) - pods, err := provider.ListAllPodsMatchingRegex(ctx, podExactRegex, []string{mizu.Config.MizuResourcesNamespace}) - if err != nil { - return err - } - - if len(pods) == 0 { - return fmt.Errorf("no mizu pods found in namespace %s", mizu.Config.MizuResourcesNamespace) - } - - newZipFile, err := os.Create(filePath) - if err != nil { - return err - } - defer newZipFile.Close() - zipWriter := zip.NewWriter(newZipFile) - defer zipWriter.Close() - - for _, pod := range pods { - logs, err := provider.GetPodLogs(pod.Namespace, pod.Name, ctx) - if err != nil { - mizu.Log.Errorf("Failed to get logs, %v", err) - continue - } else { - mizu.Log.Debugf("Successfully read log length %d for pod: %s.%s", len(logs), pod.Namespace, pod.Name) - } - if err := AddStrToZip(zipWriter, logs, fmt.Sprintf("%s.%s.log", pod.Namespace, pod.Name)); err != nil { - mizu.Log.Errorf("Failed write logs, %v", err) - } else { - mizu.Log.Infof("Successfully added log length %d from pod: %s.%s", len(logs), pod.Namespace, pod.Name) - } - } - if err := AddFileToZip(zipWriter, mizu.GetConfigFilePath()); err != nil { - mizu.Log.Debugf("Failed write file, %v", err) - } else { - mizu.Log.Infof("Successfully added file %s", mizu.GetConfigFilePath()) - } - if err := AddFileToZip(zipWriter, mizu.GetLogFilePath()); err != nil { - mizu.Log.Debugf("Failed write file, %v", err) - } else { - mizu.Log.Infof("Successfully added file %s", mizu.GetLogFilePath()) - } - mizu.Log.Infof("You can find the zip with all logs in %s\n", filePath) - return nil -} diff --git a/cli/go.mod b/cli/go.mod index 9879f012e..cfdfa96c6 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -4,6 +4,7 @@ go 1.16 require ( github.com/creasty/defaults v1.5.1 + github.com/denisbrodbeck/machineid v1.0.1 github.com/google/go-github/v37 v37.0.0 github.com/gorilla/websocket v1.4.2 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 diff --git a/cli/go.sum b/cli/go.sum index ae8e268fd..f19b35b91 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -88,6 +88,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= +github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= +github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -217,7 +219,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -411,8 +412,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index fcbb78502..1603b1307 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -7,7 +7,8 @@ import ( "encoding/json" "errors" "fmt" - "os" + "github.com/up9inc/mizu/cli/config/configStructs" + "github.com/up9inc/mizu/cli/logger" "path/filepath" "regexp" "strconv" @@ -36,7 +37,6 @@ import ( "k8s.io/client-go/tools/clientcmd" _ "k8s.io/client-go/tools/portforward" watchtools "k8s.io/client-go/tools/watch" - "k8s.io/client-go/util/homedir" ) type Provider struct { @@ -55,13 +55,23 @@ func NewProvider(kubeConfigPath string) (*Provider, error) { restClientConfig, err := kubernetesConfig.ClientConfig() if err != nil { if clientcmd.IsEmptyConfig(err) { - return nil, fmt.Errorf("Couldn't find the kube config file, or file is empty. Try adding '--kube-config='\n") + return nil, fmt.Errorf("couldn't find the kube config file, or file is empty (%s)\n" + + "you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err: %w", kubeConfigPath, err) } if clientcmd.IsConfigurationInvalid(err) { - return nil, fmt.Errorf("Invalid kube config file. Try using a different config with '--kube-config='\n") + return nil, fmt.Errorf("invalid kube config file (%s)\n" + + "you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err: %w", kubeConfigPath, err) } + + return nil, fmt.Errorf("error while using kube config (%s)\n" + + "you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err: %w", kubeConfigPath, err) + } + + clientSet, err := getClientSet(restClientConfig) + if err != nil { + return nil, fmt.Errorf("error while using kube config (%s)\n" + + "you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err: %w", kubeConfigPath, err) } - clientSet := getClientSet(restClientConfig) return &Provider{ clientSet: clientSet, @@ -140,6 +150,8 @@ type ApiServerOptions struct { IsNamespaceRestricted bool MizuApiFilteringOptions *shared.TrafficFilteringOptions MaxEntriesDBSizeBytes int64 + Resources configStructs.Resources + ImagePullPolicy core.PullPolicy } func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiServerOptions) (*core.Pod, error) { @@ -152,19 +164,19 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS configMapOptional := true configMapVolumeName.Optional = &configMapOptional - cpuLimit, err := resource.ParseQuantity("750m") + cpuLimit, err := resource.ParseQuantity(opts.Resources.CpuLimit) if err != nil { return nil, errors.New(fmt.Sprintf("invalid cpu limit for %s container", opts.PodName)) } - memLimit, err := resource.ParseQuantity("512Mi") + memLimit, err := resource.ParseQuantity(opts.Resources.MemoryLimit) if err != nil { return nil, errors.New(fmt.Sprintf("invalid memory limit for %s container", opts.PodName)) } - cpuRequests, err := resource.ParseQuantity("50m") + cpuRequests, err := resource.ParseQuantity(opts.Resources.CpuRequests) if err != nil { return nil, errors.New(fmt.Sprintf("invalid cpu request for %s container", opts.PodName)) } - memRequests, err := resource.ParseQuantity("50Mi") + memRequests, err := resource.ParseQuantity(opts.Resources.MemoryRequests) if err != nil { return nil, errors.New(fmt.Sprintf("invalid memory request for %s container", opts.PodName)) } @@ -185,7 +197,7 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS { Name: opts.PodName, Image: opts.PodImage, - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: opts.ImagePullPolicy, VolumeMounts: []core.VolumeMount{ { Name: mizu.ConfigMapName, @@ -561,8 +573,8 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, return nil } -func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool) error { - mizu.Log.Debugf("Applying %d tapper deamonsets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName) +func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool, resources configStructs.Resources, imagePullPolicy core.PullPolicy) error { + logger.Log.Debugf("Applying %d tapper deamonsets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName) if len(nodeToTappedPodIPMap) == 0 { return fmt.Errorf("Daemon set %s must tap at least 1 pod", daemonSetName) @@ -577,7 +589,6 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac "./mizuagent", "-i", "any", "--tap", - "--hardump", "--api-server-address", fmt.Sprintf("ws://%s/wsTapper", apiServerPodIp), } if tapOutgoing { @@ -587,7 +598,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac agentContainer := applyconfcore.Container() agentContainer.WithName(tapperPodName) agentContainer.WithImage(podImage) - agentContainer.WithImagePullPolicy(core.PullAlways) + agentContainer.WithImagePullPolicy(imagePullPolicy) agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(true)) agentContainer.WithCommand(mizuCmd...) agentContainer.WithEnv( @@ -601,19 +612,19 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac ), ), ) - cpuLimit, err := resource.ParseQuantity("500m") + cpuLimit, err := resource.ParseQuantity(resources.CpuLimit) if err != nil { return errors.New(fmt.Sprintf("invalid cpu limit for %s container", tapperPodName)) } - memLimit, err := resource.ParseQuantity("1Gi") + memLimit, err := resource.ParseQuantity(resources.MemoryLimit) if err != nil { return errors.New(fmt.Sprintf("invalid memory limit for %s container", tapperPodName)) } - cpuRequests, err := resource.ParseQuantity("50m") + cpuRequests, err := resource.ParseQuantity(resources.CpuRequests) if err != nil { return errors.New(fmt.Sprintf("invalid cpu request for %s container", tapperPodName)) } - memRequests, err := resource.ParseQuantity("50Mi") + memRequests, err := resource.ParseQuantity(resources.MemoryRequests) if err != nil { return errors.New(fmt.Sprintf("invalid memory request for %s container", tapperPodName)) } @@ -728,25 +739,17 @@ func (provider *Provider) GetPodLogs(namespace string, podName string, ctx conte return str, nil } -func getClientSet(config *restclient.Config) *kubernetes.Clientset { +func getClientSet(config *restclient.Config) (*kubernetes.Clientset, error) { clientSet, err := kubernetes.NewForConfig(config) if err != nil { - panic(err.Error()) + return nil, err } - return clientSet + + return clientSet, nil } func loadKubernetesConfiguration(kubeConfigPath string) clientcmd.ClientConfig { - if kubeConfigPath == "" { - kubeConfigPath = os.Getenv("KUBECONFIG") - } - - if kubeConfigPath == "" { - home := homedir.HomeDir() - kubeConfigPath = filepath.Join(home, ".kube", "config") - } - - mizu.Log.Debugf("Using kube config %s", kubeConfigPath) + logger.Log.Debugf("Using kube config %s", kubeConfigPath) configPathList := filepath.SplitList(kubeConfigPath) configLoadingRules := &clientcmd.ClientConfigLoadingRules{} if len(configPathList) <= 1 { diff --git a/cli/kubernetes/proxy.go b/cli/kubernetes/proxy.go index 5ca2eab16..44397bdc4 100644 --- a/cli/kubernetes/proxy.go +++ b/cli/kubernetes/proxy.go @@ -2,7 +2,7 @@ package kubernetes import ( "fmt" - "github.com/up9inc/mizu/cli/mizu" + "github.com/up9inc/mizu/cli/logger" "k8s.io/kubectl/pkg/proxy" "net" "net/http" @@ -14,7 +14,7 @@ const k8sProxyApiPrefix = "/" const mizuServicePort = 80 func StartProxy(kubernetesProvider *Provider, mizuPort uint16, mizuNamespace string, mizuServiceName string) error { - mizu.Log.Debugf("Starting proxy. namespace: [%v], service name: [%s], port: [%v]", mizuNamespace, mizuServiceName, mizuPort) + logger.Log.Debugf("Starting proxy. namespace: [%v], service name: [%s], port: [%v]", mizuNamespace, mizuServiceName, mizuPort) filter := &proxy.FilterServer{ AcceptPaths: proxy.MakeRegexpArrayOrDie(proxy.DefaultPathAcceptRE), RejectPaths: proxy.MakeRegexpArrayOrDie(proxy.DefaultPathRejectRE), diff --git a/cli/mizu/logger.go b/cli/logger/logger.go similarity index 83% rename from cli/mizu/logger.go rename to cli/logger/logger.go index 251922263..050ccb36d 100644 --- a/cli/mizu/logger.go +++ b/cli/logger/logger.go @@ -1,7 +1,8 @@ -package mizu +package logger import ( "github.com/op/go-logging" + "github.com/up9inc/mizu/cli/mizu" "os" "path" ) @@ -13,7 +14,7 @@ var format = logging.MustStringFormatter( ) func GetLogFilePath() string { - return path.Join(GetMizuFolderPath(), "mizu_cli.log") + return path.Join(mizu.GetMizuFolderPath(), "mizu_cli.log") } func InitLogger() { @@ -34,5 +35,5 @@ func InitLogger() { logging.SetBackend(backend1Leveled, backend2Formatter) Log.Debugf("\n\n\n") - Log.Debugf("Running mizu version %v", SemVer) + Log.Debugf("Running mizu version %v", mizu.SemVer) } diff --git a/cli/mizu.go b/cli/mizu.go index 6dc698567..05d692c66 100644 --- a/cli/mizu.go +++ b/cli/mizu.go @@ -2,7 +2,7 @@ package main import ( "github.com/up9inc/mizu/cli/cmd" - "github.com/up9inc/mizu/cli/goUtils" + "github.com/up9inc/mizu/cli/mizu/goUtils" ) func main() { diff --git a/cli/mizu/config.go b/cli/mizu/config.go index 677a4d51a..e69de29bb 100644 --- a/cli/mizu/config.go +++ b/cli/mizu/config.go @@ -1,284 +0,0 @@ -package mizu - -import ( - "errors" - "fmt" - "io/ioutil" - "os" - "path" - "reflect" - "strconv" - "strings" - - "github.com/creasty/defaults" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "github.com/up9inc/mizu/cli/mizu/configStructs" - "github.com/up9inc/mizu/cli/uiUtils" - "gopkg.in/yaml.v3" -) - -const ( - Separator = "=" - SetCommandName = "set" -) - -var allowedSetFlags = []string{ - AgentImageConfigName, - MizuResourcesNamespaceConfigName, - TelemetryConfigName, - DumpLogsConfigName, - KubeConfigPathName, - configStructs.AnalysisDestinationTapName, - configStructs.SleepIntervalSecTapName, -} - -var Config = ConfigStruct{} - -func (config *ConfigStruct) Validate() error { - if config.IsNsRestrictedMode() { - if config.Tap.AllNamespaces || len(config.Tap.Namespaces) != 1 || config.Tap.Namespaces[0] != config.MizuResourcesNamespace { - return fmt.Errorf("Not supported mode. Mizu can't resolve IPs in other namespaces when running in namespace restricted mode.\n"+ - "You can use the same namespace for --%s and --%s", configStructs.NamespacesTapName, MizuResourcesNamespaceConfigName) - } - } - - return nil -} - -func InitConfig(cmd *cobra.Command) error { - if err := defaults.Set(&Config); err != nil { - return err - } - - if err := mergeConfigFile(); err != nil { - return fmt.Errorf("invalid config %w\n"+ - "you can regenerate the file using `mizu config -r` or just remove it %v", err, GetConfigFilePath()) - } - - cmd.Flags().Visit(initFlag) - - finalConfigPrettified, _ := uiUtils.PrettyJson(Config) - Log.Debugf("Init config finished\n Final config: %v", finalConfigPrettified) - - return nil -} - -func GetConfigWithDefaults() (string, error) { - defaultConf := ConfigStruct{} - if err := defaults.Set(&defaultConf); err != nil { - return "", err - } - - // TODO: change to generic solution - defaultConf.AgentImage = "" - - return uiUtils.PrettyYaml(defaultConf) -} - -func GetConfigFilePath() string { - return path.Join(GetMizuFolderPath(), "config.yaml") -} - -func mergeConfigFile() error { - reader, openErr := os.Open(GetConfigFilePath()) - if openErr != nil { - return nil - } - - buf, readErr := ioutil.ReadAll(reader) - if readErr != nil { - return readErr - } - - if err := yaml.Unmarshal(buf, &Config); err != nil { - return err - } - Log.Debugf("Found config file, merged to default options") - - return nil -} - -func initFlag(f *pflag.Flag) { - configElem := reflect.ValueOf(&Config).Elem() - - sliceValue, isSliceValue := f.Value.(pflag.SliceValue) - if !isSliceValue { - mergeFlagValue(configElem, f.Name, f.Value.String()) - return - } - - if f.Name == SetCommandName { - mergeSetFlag(sliceValue.GetSlice()) - return - } - - mergeFlagValues(configElem, f.Name, sliceValue.GetSlice()) -} - -func mergeSetFlag(setValues []string) { - configElem := reflect.ValueOf(&Config).Elem() - - for _, setValue := range setValues { - if !strings.Contains(setValue, Separator) { - Log.Warningf(uiUtils.Warning, fmt.Sprintf("Ignoring set argument %s (set argument format: =)", setValue)) - } - - split := strings.SplitN(setValue, Separator, 2) - if len(split) != 2 { - Log.Warningf(uiUtils.Warning, fmt.Sprintf("Ignoring set argument %s (set argument format: =)", setValue)) - } - - argumentKey, argumentValue := split[0], split[1] - - if !Contains(allowedSetFlags, argumentKey) { - Log.Warningf(uiUtils.Warning, fmt.Sprintf("Ignoring set argument %s, flag name must be one of the following: \"%s\"", setValue, strings.Join(allowedSetFlags, "\", \""))) - continue - } - - mergeFlagValue(configElem, argumentKey, argumentValue) - } -} - -func mergeFlagValue(currentElem reflect.Value, flagKey string, flagValue string) { - for i := 0; i < currentElem.NumField(); i++ { - currentField := currentElem.Type().Field(i) - currentFieldByName := currentElem.FieldByName(currentField.Name) - - if currentField.Type.Kind() == reflect.Struct { - mergeFlagValue(currentFieldByName, flagKey, flagValue) - continue - } - - if currentField.Tag.Get("yaml") != flagKey { - continue - } - - flagValueKind := currentField.Type.Kind() - - parsedValue, err := getParsedValue(flagValueKind, flagValue) - if err != nil { - Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for flag name %s, expected %s", flagValue, flagKey, flagValueKind)) - return - } - - currentFieldByName.Set(parsedValue) - } -} - -func mergeFlagValues(currentElem reflect.Value, flagKey string, flagValues []string) { - for i := 0; i < currentElem.NumField(); i++ { - currentField := currentElem.Type().Field(i) - currentFieldByName := currentElem.FieldByName(currentField.Name) - - if currentField.Type.Kind() == reflect.Struct { - mergeFlagValues(currentFieldByName, flagKey, flagValues) - continue - } - - if currentField.Tag.Get("yaml") != flagKey { - continue - } - - flagValueKind := currentField.Type.Elem().Kind() - - parsedValues := reflect.MakeSlice(reflect.SliceOf(currentField.Type.Elem()), 0, 0) - for _, flagValue := range flagValues { - parsedValue, err := getParsedValue(flagValueKind, flagValue) - if err != nil { - Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for flag name %s, expected %s", flagValue, flagKey, flagValueKind)) - return - } - - parsedValues = reflect.Append(parsedValues, parsedValue) - } - - currentFieldByName.Set(parsedValues) - } -} - -func getParsedValue(kind reflect.Kind, value string) (reflect.Value, error) { - switch kind { - case reflect.String: - return reflect.ValueOf(value), nil - case reflect.Bool: - boolArgumentValue, err := strconv.ParseBool(value) - if err != nil { - break - } - - return reflect.ValueOf(boolArgumentValue), nil - case reflect.Int: - intArgumentValue, err := strconv.ParseInt(value, 10, 64) - if err != nil { - break - } - - return reflect.ValueOf(int(intArgumentValue)), nil - case reflect.Int8: - intArgumentValue, err := strconv.ParseInt(value, 10, 8) - if err != nil { - break - } - - return reflect.ValueOf(int8(intArgumentValue)), nil - case reflect.Int16: - intArgumentValue, err := strconv.ParseInt(value, 10, 16) - if err != nil { - break - } - - return reflect.ValueOf(int16(intArgumentValue)), nil - case reflect.Int32: - intArgumentValue, err := strconv.ParseInt(value, 10, 32) - if err != nil { - break - } - - return reflect.ValueOf(int32(intArgumentValue)), nil - case reflect.Int64: - intArgumentValue, err := strconv.ParseInt(value, 10, 64) - if err != nil { - break - } - - return reflect.ValueOf(intArgumentValue), nil - case reflect.Uint: - uintArgumentValue, err := strconv.ParseUint(value, 10, 64) - if err != nil { - break - } - - return reflect.ValueOf(uint(uintArgumentValue)), nil - case reflect.Uint8: - uintArgumentValue, err := strconv.ParseUint(value, 10, 8) - if err != nil { - break - } - - return reflect.ValueOf(uint8(uintArgumentValue)), nil - case reflect.Uint16: - uintArgumentValue, err := strconv.ParseUint(value, 10, 16) - if err != nil { - break - } - - return reflect.ValueOf(uint16(uintArgumentValue)), nil - case reflect.Uint32: - uintArgumentValue, err := strconv.ParseUint(value, 10, 32) - if err != nil { - break - } - - return reflect.ValueOf(uint32(uintArgumentValue)), nil - case reflect.Uint64: - uintArgumentValue, err := strconv.ParseUint(value, 10, 64) - if err != nil { - break - } - - return reflect.ValueOf(uintArgumentValue), nil - } - - return reflect.ValueOf(nil), errors.New("value to parse does not match type") -} diff --git a/cli/mizu/configStructs/viewConfig.go b/cli/mizu/configStructs/viewConfig.go deleted file mode 100644 index 6f26ca9dc..000000000 --- a/cli/mizu/configStructs/viewConfig.go +++ /dev/null @@ -1,11 +0,0 @@ -package configStructs - -const ( - GuiPortViewName = "gui-port" - KubeConfigPathViewName = "kube-config" -) - -type ViewConfig struct { - GuiPort uint16 `yaml:"gui-port" default:"8899"` - KubeConfigPath string `yaml:"kube-config"` -} diff --git a/cli/fsUtils/dirUtils.go b/cli/mizu/fsUtils/dirUtils.go similarity index 100% rename from cli/fsUtils/dirUtils.go rename to cli/mizu/fsUtils/dirUtils.go diff --git a/cli/mizu/fsUtils/mizuLogsUtils.go b/cli/mizu/fsUtils/mizuLogsUtils.go new file mode 100644 index 000000000..5edb5d41b --- /dev/null +++ b/cli/mizu/fsUtils/mizuLogsUtils.go @@ -0,0 +1,60 @@ +package fsUtils + +import ( + "archive/zip" + "context" + "fmt" + "github.com/up9inc/mizu/cli/config" + "github.com/up9inc/mizu/cli/kubernetes" + "github.com/up9inc/mizu/cli/logger" + "github.com/up9inc/mizu/cli/mizu" + "os" + "regexp" +) + +func DumpLogs(provider *kubernetes.Provider, ctx context.Context, filePath string) error { + podExactRegex := regexp.MustCompile("^" + mizu.MizuResourcesPrefix) + pods, err := provider.ListAllPodsMatchingRegex(ctx, podExactRegex, []string{config.Config.MizuResourcesNamespace}) + if err != nil { + return err + } + + if len(pods) == 0 { + return fmt.Errorf("no mizu pods found in namespace %s", config.Config.MizuResourcesNamespace) + } + + newZipFile, err := os.Create(filePath) + if err != nil { + return err + } + defer newZipFile.Close() + zipWriter := zip.NewWriter(newZipFile) + defer zipWriter.Close() + + for _, pod := range pods { + logs, err := provider.GetPodLogs(pod.Namespace, pod.Name, ctx) + if err != nil { + logger.Log.Errorf("Failed to get logs, %v", err) + continue + } else { + logger.Log.Debugf("Successfully read log length %d for pod: %s.%s", len(logs), pod.Namespace, pod.Name) + } + if err := AddStrToZip(zipWriter, logs, fmt.Sprintf("%s.%s.log", pod.Namespace, pod.Name)); err != nil { + logger.Log.Errorf("Failed write logs, %v", err) + } else { + logger.Log.Debugf("Successfully added log length %d from pod: %s.%s", len(logs), pod.Namespace, pod.Name) + } + } + if err := AddFileToZip(zipWriter, config.GetConfigFilePath()); err != nil { + logger.Log.Debugf("Failed write file, %v", err) + } else { + logger.Log.Debugf("Successfully added file %s", config.GetConfigFilePath()) + } + if err := AddFileToZip(zipWriter, logger.GetLogFilePath()); err != nil { + logger.Log.Debugf("Failed write file, %v", err) + } else { + logger.Log.Debugf("Successfully added file %s", logger.GetLogFilePath()) + } + logger.Log.Infof("You can find the zip file with all logs in %s\n", filePath) + return nil +} diff --git a/cli/fsUtils/zipUtils.go b/cli/mizu/fsUtils/zipUtils.go similarity index 100% rename from cli/fsUtils/zipUtils.go rename to cli/mizu/fsUtils/zipUtils.go diff --git a/cli/goUtils/funcWrappers.go b/cli/mizu/goUtils/funcWrappers.go similarity index 81% rename from cli/goUtils/funcWrappers.go rename to cli/mizu/goUtils/funcWrappers.go index ad71b684b..fd1f5b1ce 100644 --- a/cli/goUtils/funcWrappers.go +++ b/cli/mizu/goUtils/funcWrappers.go @@ -1,7 +1,7 @@ package goUtils import ( - "github.com/up9inc/mizu/cli/mizu" + "github.com/up9inc/mizu/cli/logger" "reflect" "runtime/debug" ) @@ -10,7 +10,7 @@ func HandleExcWrapper(fn interface{}, params ...interface{}) (result []reflect.V defer func() { if panicMessage := recover(); panicMessage != nil { stack := debug.Stack() - mizu.Log.Fatalf("Unhandled panic: %v\n stack: %s", panicMessage, stack) + logger.Log.Fatalf("Unhandled panic: %v\n stack: %s", panicMessage, stack) } }() f := reflect.ValueOf(fn) diff --git a/cli/mizu/sliceUtils_test.go b/cli/mizu/sliceUtils_test.go new file mode 100644 index 000000000..49787c64c --- /dev/null +++ b/cli/mizu/sliceUtils_test.go @@ -0,0 +1,90 @@ +package mizu_test + +import ( + "github.com/up9inc/mizu/cli/mizu" + "testing" +) + +func TestContainsExists(t *testing.T) { + tests := []struct { + Slice []string + ContainsValue string + Expected bool + }{ + {Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "apple", Expected: true}, + {Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "orange", Expected: true}, + {Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "banana", Expected: true}, + {Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "grapes", Expected: true}, + } + + for _, test := range tests { + t.Run(test.ContainsValue, func(t *testing.T) { + actual := mizu.Contains(test.Slice, test.ContainsValue) + if actual != test.Expected { + t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual) + } + }) + } +} + +func TestContainsNotExists(t *testing.T) { + tests := []struct { + Slice []string + ContainsValue string + Expected bool + }{ + {Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "cat", Expected: false}, + {Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "dog", Expected: false}, + {Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "apples", Expected: false}, + {Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "rapes", Expected: false}, + } + + for _, test := range tests { + t.Run(test.ContainsValue, func(t *testing.T) { + actual := mizu.Contains(test.Slice, test.ContainsValue) + if actual != test.Expected { + t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual) + } + }) + } +} + +func TestContainsEmptySlice(t *testing.T) { + tests := []struct { + Slice []string + ContainsValue string + Expected bool + }{ + {Slice: []string{}, ContainsValue: "cat", Expected: false}, + {Slice: []string{}, ContainsValue: "dog", Expected: false}, + } + + for _, test := range tests { + t.Run(test.ContainsValue, func(t *testing.T) { + actual := mizu.Contains(test.Slice, test.ContainsValue) + if actual != test.Expected { + t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual) + } + }) + } +} + +func TestContainsNilSlice(t *testing.T) { + tests := []struct { + Slice []string + ContainsValue string + Expected bool + }{ + {Slice: nil, ContainsValue: "cat", Expected: false}, + {Slice: nil, ContainsValue: "dog", Expected: false}, + } + + for _, test := range tests { + t.Run(test.ContainsValue, func(t *testing.T) { + actual := mizu.Contains(test.Slice, test.ContainsValue) + if actual != test.Expected { + t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual) + } + }) + } +} diff --git a/cli/mizu/telemetry.go b/cli/mizu/telemetry.go deleted file mode 100644 index e36956cd9..000000000 --- a/cli/mizu/telemetry.go +++ /dev/null @@ -1,36 +0,0 @@ -package mizu - -import ( - "bytes" - "encoding/json" - "fmt" - "net/http" -) - -const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry" - -func ReportRun(cmd string, args interface{}) { - if !Config.Telemetry { - Log.Debugf("not reporting due to config value") - return - } - - argsBytes, _ := json.Marshal(args) - argsMap := map[string]string{ - "telemetry_type": "execution", - "cmd": cmd, - "args": string(argsBytes), - "component": "mizu_cli", - "BuildTimestamp": BuildTimestamp, - "Branch": Branch, - "version": SemVer} - argsMap["message"] = fmt.Sprintf("mizu %v - %v", argsMap["cmd"], string(argsBytes)) - - jsonValue, _ := json.Marshal(argsMap) - - if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil { - Log.Debugf("error sending telemetry err: %v, response %v", err, resp) - } else { - Log.Debugf("Successfully reported telemetry") - } -} diff --git a/cli/mizu/versionCheck.go b/cli/mizu/version/versionCheck.go similarity index 56% rename from cli/mizu/versionCheck.go rename to cli/mizu/version/versionCheck.go index 74a5c84e3..eb479dd7a 100644 --- a/cli/mizu/versionCheck.go +++ b/cli/mizu/version/versionCheck.go @@ -1,9 +1,11 @@ -package mizu +package version import ( "context" "encoding/json" "fmt" + "github.com/up9inc/mizu/cli/logger" + "github.com/up9inc/mizu/cli/mizu" "io/ioutil" "net/http" "net/url" @@ -41,22 +43,22 @@ func CheckVersionCompatibility(port uint16) (bool, error) { return false, err } - if semver.SemVersion(apiSemVer).Major() == semver.SemVersion(SemVer).Major() && - semver.SemVersion(apiSemVer).Minor() == semver.SemVersion(SemVer).Minor() { + if semver.SemVersion(apiSemVer).Major() == semver.SemVersion(mizu.SemVer).Major() && + semver.SemVersion(apiSemVer).Minor() == semver.SemVersion(mizu.SemVer).Minor() { return true, nil } - Log.Errorf(uiUtils.Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)", SemVer, apiSemVer)) + logger.Log.Errorf(uiUtils.Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)", mizu.SemVer, apiSemVer)) return false, nil } -func CheckNewerVersion() { - Log.Debugf("Checking for newer version...") +func CheckNewerVersion(versionChan chan string) { + logger.Log.Debugf("Checking for newer version...") start := time.Now() client := github.NewClient(nil) latestRelease, _, err := client.Repositories.GetLatestRelease(context.Background(), "up9inc", "mizu") if err != nil { - Log.Debugf("[ERROR] Failed to get latest release") + logger.Log.Debugf("[ERROR] Failed to get latest release") return } @@ -68,26 +70,31 @@ func CheckNewerVersion() { } } if versionFileUrl == "" { - Log.Debugf("[ERROR] Version file not found in the latest release") + logger.Log.Debugf("[ERROR] Version file not found in the latest release") return } res, err := http.Get(versionFileUrl) if err != nil { - Log.Debugf("[ERROR] Failed to get the version file %v", err) + logger.Log.Debugf("[ERROR] Failed to get the version file %v", err) return } data, err := ioutil.ReadAll(res.Body) res.Body.Close() if err != nil { - Log.Debugf("[ERROR] Failed to read the version file -> %v", err) + logger.Log.Debugf("[ERROR] Failed to read the version file -> %v", err) return } gitHubVersion := string(data) gitHubVersion = gitHubVersion[:len(gitHubVersion)-1] - Log.Debugf("Finished version validation, took %v", time.Since(start)) - if SemVer < gitHubVersion { - Log.Infof(uiUtils.Yellow, fmt.Sprintf("Update available! %v -> %v (%v)", SemVer, gitHubVersion, *latestRelease.HTMLURL)) + + gitHubVersionSemVer := semver.SemVersion(gitHubVersion) + currentSemVer := semver.SemVersion(mizu.SemVer) + logger.Log.Debugf("Finished version validation, github version %v, current version %v, took %v", gitHubVersion, currentSemVer, time.Since(start)) + + if gitHubVersionSemVer.GreaterThan(currentSemVer) { + versionChan <- fmt.Sprintf("Update available! %v -> %v (%v)", mizu.SemVer, gitHubVersion, *latestRelease.HTMLURL) } + versionChan <- "" } diff --git a/cli/telemetry/telemetry.go b/cli/telemetry/telemetry.go new file mode 100644 index 000000000..ffa65771b --- /dev/null +++ b/cli/telemetry/telemetry.go @@ -0,0 +1,114 @@ +package telemetry + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/denisbrodbeck/machineid" + "github.com/up9inc/mizu/cli/config" + "github.com/up9inc/mizu/cli/kubernetes" + "github.com/up9inc/mizu/cli/logger" + "github.com/up9inc/mizu/cli/mizu" + "io/ioutil" + "net/http" +) + +const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry" + +func ReportRun(cmd string, args interface{}) { + if !shouldRunTelemetry() { + logger.Log.Debugf("not reporting telemetry") + return + } + + argsBytes, _ := json.Marshal(args) + argsMap := map[string]interface{}{ + "cmd": cmd, + "args": string(argsBytes), + } + + if err := sendTelemetry("Execution", argsMap); err != nil { + logger.Log.Debug(err) + return + } + + logger.Log.Debugf("successfully reported telemetry for cmd %v", cmd) +} + +func ReportAPICalls(mizuPort uint16) { + if !shouldRunTelemetry() { + logger.Log.Debugf("not reporting telemetry") + return + } + + mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizuPort) + generalStatsUrl := fmt.Sprintf("http://%s/api/generalStats", mizuProxiedUrl) + + response, requestErr := http.Get(generalStatsUrl) + if requestErr != nil { + logger.Log.Debugf("ERROR: failed to get general stats for telemetry, err: %v", requestErr) + return + } else if response.StatusCode != 200 { + logger.Log.Debugf("ERROR: failed to get general stats for telemetry, status code: %v", response.StatusCode) + return + } + + defer func() { _ = response.Body.Close() }() + + data, readErr := ioutil.ReadAll(response.Body) + if readErr != nil { + logger.Log.Debugf("ERROR: failed to read general stats for telemetry, err: %v", readErr) + return + } + + var generalStats map[string]interface{} + if parseErr := json.Unmarshal(data, &generalStats); parseErr != nil { + logger.Log.Debugf("ERROR: failed to parse general stats for telemetry, err: %v", parseErr) + return + } + + argsMap := map[string]interface{}{ + "apiCallsCount": generalStats["EntriesCount"], + "firstAPICallTimestamp": generalStats["FirstEntryTimestamp"], + "lastAPICallTimestamp": generalStats["LastEntryTimestamp"], + } + + if err := sendTelemetry("APICalls", argsMap); err != nil { + logger.Log.Debug(err) + return + } + + logger.Log.Debugf("successfully reported telemetry of api calls") +} + +func shouldRunTelemetry() bool { + if !config.Config.Telemetry { + return false + } + + if mizu.Branch != "main" && mizu.Branch != "develop" { + return false + } + + return true +} + +func sendTelemetry(telemetryType string, argsMap map[string]interface{}) error { + argsMap["telemetryType"] = telemetryType + argsMap["component"] = "mizu_cli" + argsMap["buildTimestamp"] = mizu.BuildTimestamp + argsMap["branch"] = mizu.Branch + argsMap["version"] = mizu.SemVer + + if machineId, err := machineid.ProtectedID("mizu"); err == nil { + argsMap["machineId"] = machineId + } + + jsonValue, _ := json.Marshal(argsMap) + + if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil { + return fmt.Errorf("ERROR: failed sending telemetry, err: %v, response %v", err, resp) + } + + return nil +} diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..be82d548a --- /dev/null +++ b/codecov.yml @@ -0,0 +1,8 @@ +coverage: + status: + project: + default: + threshold: 1% + patch: + default: + enabled: no diff --git a/docs/POLICY_RULES.md b/docs/POLICY_RULES.md new file mode 100644 index 000000000..dce7257b6 --- /dev/null +++ b/docs/POLICY_RULES.md @@ -0,0 +1,78 @@ + +# API rules validation + +This feature allows you to define set of simple rules, and test the API against them. +Such validation may test response for specific JSON fields, headers, etc. + +## Examples + + +Example 1: HTTP request (REST API call) that didn’t pass validation is highlighted in red + +![Simple UI](../assets/validation-example1.png) + +- - - + + +Example 2: Details pane shows the validation rule details and whether it passed or failed + +![Simple UI](../assets/validation-example2.png) + + +## How to use +To use this feature - create simple rules file (see details below) and pass this file as parameter to `mizu tap` command. For example, if rules are stored in file named `rules.yaml` — run the following command: + + +```shell +mizu tap --test-rules rules.yaml PODNAME +``` + + + +## Rules file structure + +The structure of the test-rules-file is: + +* `name`: string, name of the rule +* `type`: string, type of the rule, must be `json` or `header` or `latency` +* `key`: string, [jsonpath](https://code.google.com/archive/p/jsonpath/wikis/Javascript.wiki) used only in `json` or `header` type +* `value`: string, [regex](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) used only in `json` or `header` type +* `service`: string, [regex](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) service name to filter +* `path`: string, [regex](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) URL path to filter +* `latency`: integer, time in ms of the expected latency. + + +### For example: + +```yaml +rules: +- name: holy-in-name-property + type: json + key: "$.name" + value: "Holy" + service: "catalogue.*" + path: "catalogue.*" +- name: content-length-header + type: header + key: "Content-Le.*" + value: "(\\d+(?:\\.\\d+)?)" +- name: latency-test + type: latency + latency: 1 + service: "carts.*" +``` + +### Explanation: + +* First rule `holy-in-name-property`: + + > This rule will be applied to all request made to `catalogue.*` services with `catalogue.*` on the URL path with a json response containing a `$.name` field. If the value of `$.name` is `Holy` than is marked as success, marked as failure otherwise. + +* Second rule `content-length-header`: + + > This rule will be applied to all request that has `Content-Le.*` on header. If the value of `Content-Le.*` is `(\\d+(?:\\.\\d+)?)` (number), will be marked as success, marked as failure otherwise. + +* Third rule `latency-test`: + + > This rule will be applied to all request made to `carts.*` services. If the latency of the response is greater than `1` will be marked as failure, marked as success otherwise. + diff --git a/shared/go.mod b/shared/go.mod index 157d3e5fa..5e50ad375 100644 --- a/shared/go.mod +++ b/shared/go.mod @@ -3,8 +3,7 @@ module github.com/up9inc/mizu/shared go 1.16 require ( - github.com/google/martian v2.1.0+incompatible // indirect - github.com/gorilla/websocket v1.4.2 - github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect github.com/docker/go-units v0.4.0 + github.com/gorilla/websocket v1.4.2 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/shared/go.sum b/shared/go.sum index 498bce1d3..65c692cb5 100644 --- a/shared/go.sum +++ b/shared/go.sum @@ -1,8 +1,8 @@ -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/shared/models.go b/shared/models.go index 2a70fa212..ade4a9269 100644 --- a/shared/models.go +++ b/shared/models.go @@ -75,9 +75,9 @@ func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSoc } type TrafficFilteringOptions struct { - PlainTextMaskingRegexes []*SerializableRegexp - HideHealthChecks bool - DisableRedaction bool + HealthChecksUserAgentHeaders []string + PlainTextMaskingRegexes []*SerializableRegexp + DisableRedaction bool } type VersionResponse struct { diff --git a/shared/semver/semver.go b/shared/semver/semver.go index f27ca33f0..441b610fb 100644 --- a/shared/semver/semver.go +++ b/shared/semver/semver.go @@ -26,3 +26,23 @@ func (v SemVersion) Patch() string { _, _, patch := v.Breakdown() return patch } + +func (v SemVersion) GreaterThan(v2 SemVersion) bool { + if v.Major() > v2.Major() { + return true + } else if v.Major() < v2.Major() { + return false + } + + if v.Minor() > v2.Minor() { + return true + } else if v.Minor() < v2.Minor() { + return false + } + + if v.Patch() > v2.Patch() { + return true + } + + return false +} diff --git a/tap/http_reader.go b/tap/http_reader.go index da7266a8b..be6652e97 100644 --- a/tap/http_reader.go +++ b/tap/http_reader.go @@ -80,7 +80,7 @@ func (h *httpReader) Read(p []byte) (int, error) { err := clientHello.Unmarshall(msg.bytes) if err == nil { statsTracker.incTlsConnectionsCount() - fmt.Printf("Detected TLS client hello with SNI %s\n", clientHello.SNI) + Debug("Detected TLS client hello with SNI %s\n", clientHello.SNI) numericPort, _ := strconv.Atoi(h.tcpID.dstPort) h.outboundLinkWriter.WriteOutboundLink(h.tcpID.srcIP, h.tcpID.dstIP, numericPort, clientHello.SNI, TLSProtocol) } diff --git a/tap/passive_tapper.go b/tap/passive_tapper.go index afb0ee37d..cc0db564c 100644 --- a/tap/passive_tapper.go +++ b/tap/passive_tapper.go @@ -86,7 +86,6 @@ var staleTimeoutSeconds = flag.Int("staletimout", 120, "Max time in seconds to k var memprofile = flag.String("memprofile", "", "Write memory profile") // output -var dumpToHar = flag.Bool("hardump", false, "Dump traffic to har files") var HarOutputDir = flag.String("hardir", "", "Directory in which to store output har files") var harEntriesPerFile = flag.Int("harentriesperfile", 200, "Number of max number of har entries to store in each file") @@ -188,19 +187,12 @@ func (c *Context) GetCaptureInfo() gopacket.CaptureInfo { func StartPassiveTapper(opts *TapOpts) (<-chan *OutputChannelItem, <-chan *OutboundLink) { hostMode = opts.HostMode - var harWriter *HarWriter - if *dumpToHar { - harWriter = NewHarWriter(*HarOutputDir, *harEntriesPerFile) - } + harWriter := NewHarWriter(*HarOutputDir, *harEntriesPerFile) outboundLinkWriter := NewOutboundLinkWriter() go startPassiveTapper(harWriter, outboundLinkWriter) - if harWriter != nil { - return harWriter.OutChan, outboundLinkWriter.OutChan - } - - return nil, outboundLinkWriter.OutChan + return harWriter.OutChan, outboundLinkWriter.OutChan } func startMemoryProfiler() { @@ -323,10 +315,8 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr } } - if *dumpToHar { - harWriter.Start() - defer harWriter.Stop() - } + harWriter.Start() + defer harWriter.Stop() defer outboundLinkWriter.Stop() var dec gopacket.Decoder diff --git a/ui/src/App.sass b/ui/src/App.sass index 629011eac..0b409a236 100644 --- a/ui/src/App.sass +++ b/ui/src/App.sass @@ -1,4 +1,4 @@ -@import 'components/style/variables.module' +@import 'src/variables.module' .mizuApp background-color: $main-background-color diff --git a/ui/src/App.tsx b/ui/src/App.tsx index c30d09083..fa5b8153c 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -2,8 +2,8 @@ import React, {useEffect, useState} from 'react'; import './App.sass'; import logo from './components/assets/Mizu-logo.svg'; import {Button, Snackbar} from "@material-ui/core"; -import {HarPage} from "./components/HarPage"; -import Tooltip from "./components/Tooltip"; +import {TrafficPage} from "./components/TrafficPage"; +import Tooltip from "./components/UI/Tooltip"; import {makeStyles} from "@material-ui/core/styles"; import MuiAlert from '@material-ui/lab/Alert'; import Api from "./helpers/api"; @@ -38,6 +38,7 @@ const App = () => { } })(); + // eslint-disable-next-line }, []); const onTLSDetected = (destAddress: string) => { @@ -116,7 +117,7 @@ const App = () => { } - + setUserDismissedTLSWarning(true)} severity="warning"> Mizu is detecting TLS traffic{addressesWithTLS.size ? ` (directed to ${Array.from(addressesWithTLS).join(", ")})` : ''}, this type of traffic will not be displayed. diff --git a/ui/src/components/HarEntriesList.tsx b/ui/src/components/EntriesList.tsx similarity index 76% rename from ui/src/components/HarEntriesList.tsx rename to ui/src/components/EntriesList.tsx index cea3c2cef..31c2634f3 100644 --- a/ui/src/components/HarEntriesList.tsx +++ b/ui/src/components/EntriesList.tsx @@ -1,16 +1,17 @@ -import {HarEntry} from "./HarEntry"; -import React, {useCallback, useEffect, useMemo, useState} from "react"; -import styles from './style/HarEntriesList.module.sass'; +import {EntryItem} from "./EntryListItem/EntryListItem"; +import React, {useCallback, useEffect, useMemo, useRef, useState} from "react"; +import styles from './style/EntriesList.module.sass'; import spinner from './assets/spinner.svg'; import ScrollableFeed from "react-scrollable-feed"; -import {StatusType} from "./HarFilters"; +import {StatusType} from "./Filters"; import Api from "../helpers/api"; +import down from "./assets/downImg.svg"; interface HarEntriesListProps { entries: any[]; setEntries: (entries: any[]) => void; - focusedEntryId: string; - setFocusedEntryId: (id: string) => void; + focusedEntry: any; + setFocusedEntry: (entry: any) => void; connectionOpen: boolean; noMoreDataTop: boolean; setNoMoreDataTop: (flag: boolean) => void; @@ -19,6 +20,9 @@ interface HarEntriesListProps { methodsFilter: Array; statusFilter: Array; pathFilter: string + listEntryREF: any; + onScrollEvent: (isAtBottom:boolean) => void; + scrollableList: boolean; } enum FetchOperator { @@ -28,11 +32,12 @@ enum FetchOperator { const api = new Api(); -export const HarEntriesList: React.FC = ({entries, setEntries, focusedEntryId, setFocusedEntryId, connectionOpen, noMoreDataTop, setNoMoreDataTop, noMoreDataBottom, setNoMoreDataBottom, methodsFilter, statusFilter, pathFilter}) => { +export const EntriesList: React.FC = ({entries, setEntries, focusedEntry, setFocusedEntry, connectionOpen, noMoreDataTop, setNoMoreDataTop, noMoreDataBottom, setNoMoreDataBottom, methodsFilter, statusFilter, pathFilter, listEntryREF, onScrollEvent, scrollableList}) => { const [loadMoreTop, setLoadMoreTop] = useState(false); const [isLoadingTop, setIsLoadingTop] = useState(false); - + const scrollableRef = useRef(null); + useEffect(() => { const list = document.getElementById('list').firstElementChild; list.addEventListener('scroll', (e) => { @@ -106,20 +111,25 @@ export const HarEntriesList: React.FC = ({entries, setEntri return <>
-
+
{isLoadingTop &&
spinner
} - + onScrollEvent(isAtBottom)}> {noMoreDataTop && !connectionOpen &&
No more data available
} - {filteredEntries.map(entry => )} + setFocusedEntry = {setFocusedEntry} + isSelected={focusedEntry.id === entry.id}/>)} {!connectionOpen && !noMoreDataBottom &&
getNewEntries()}>Fetch more entries
} +
{entries?.length > 0 &&
diff --git a/ui/src/components/EntryDetailed/EntryDetailed.module.sass b/ui/src/components/EntryDetailed/EntryDetailed.module.sass new file mode 100644 index 000000000..2af3d6a54 --- /dev/null +++ b/ui/src/components/EntryDetailed/EntryDetailed.module.sass @@ -0,0 +1,23 @@ +@import "src/variables.module" + +.content + font-family: "Source Sans Pro", Lucida Grande, Tahoma, sans-serif + height: calc(100% - 56px) + overflow-y: auto + width: 100% + + .body + background: $main-background-color + color: $blue-gray + border-radius: 4px + padding: 10px + .bodyHeader + padding: 0 1rem + .endpointURL + font-size: .75rem + display: block + color: $blue-color + text-decoration: none + margin-bottom: .5rem + overflow-wrap: anywhere + padding: 5px 0 \ No newline at end of file diff --git a/ui/src/components/EntryDetailed/EntryDetailed.tsx b/ui/src/components/EntryDetailed/EntryDetailed.tsx new file mode 100644 index 000000000..0db1d1a6a --- /dev/null +++ b/ui/src/components/EntryDetailed/EntryDetailed.tsx @@ -0,0 +1,56 @@ +import React from "react"; +import styles from './EntryDetailed.module.sass'; +import {makeStyles} from "@material-ui/core"; +import {EntryType} from "../EntryListItem/EntryListItem"; +import {RestEntryDetailsTitle} from "./Rest/RestEntryDetailsTitle"; +import {KafkaEntryDetailsTitle} from "./Kafka/KafkaEntryDetailsTitle"; +import {RestEntryDetailsContent} from "./Rest/RestEntryDetailsContent"; +import {KafkaEntryDetailsContent} from "./Kafka/KafkaEntryDetailsContent"; + +const useStyles = makeStyles(() => ({ + entryTitle: { + display: 'flex', + minHeight: 46, + maxHeight: 46, + alignItems: 'center', + marginBottom: 8, + padding: 5, + paddingBottom: 0 + } +})); + +interface EntryDetailedProps { + entryData: any; + classes?: any; + entryType: string; +} + +export const EntryDetailed: React.FC = ({classes, entryData, entryType}) => { + const classesTitle = useStyles(); + + let title, content; + + switch (entryType) { + case EntryType.Rest: + title = ; + content = ; + break; + case EntryType.Kafka: + title = ; + content = ; + break; + default: + title = ; + content = ; + break; + } + + return <> +
{title}
+
+
+ {content} +
+
+ +}; \ No newline at end of file diff --git a/ui/src/components/HarEntryViewer/HAREntrySections.module.sass b/ui/src/components/EntryDetailed/EntrySections.module.sass similarity index 96% rename from ui/src/components/HarEntryViewer/HAREntrySections.module.sass rename to ui/src/components/EntryDetailed/EntrySections.module.sass index 50bb0cc19..f6d73bc7c 100644 --- a/ui/src/components/HarEntryViewer/HAREntrySections.module.sass +++ b/ui/src/components/EntryDetailed/EntrySections.module.sass @@ -1,4 +1,4 @@ -@import '../style/variables.module' +@import 'src/variables.module' .title display: flex @@ -92,3 +92,6 @@ tr td:first-child white-space: nowrap padding-right: .5rem + +.noRules + padding: 0 1rem 1rem diff --git a/ui/src/components/EntryDetailed/EntrySections.tsx b/ui/src/components/EntryDetailed/EntrySections.tsx new file mode 100644 index 000000000..c6546f0a4 --- /dev/null +++ b/ui/src/components/EntryDetailed/EntrySections.tsx @@ -0,0 +1,213 @@ +import styles from "./EntrySections.module.sass"; +import React, {useState} from "react"; +import {SyntaxHighlighter} from "../UI/SyntaxHighlighter"; +import CollapsibleContainer from "../UI/CollapsibleContainer"; +import FancyTextDisplay from "../UI/FancyTextDisplay"; +import Checkbox from "../UI/Checkbox"; +import ProtobufDecoder from "protobuf-decoder"; + +interface ViewLineProps { + label: string; + value: number | string; +} + +const ViewLine: React.FC = ({label, value}) => { + return (label && value && + {label} + + + + ) || null; +} + +interface SectionCollapsibleTitleProps { + title: string; + isExpanded: boolean; +} + +const SectionCollapsibleTitle: React.FC = ({title, isExpanded}) => { + return
+ + {isExpanded ? '-' : '+'} + + {title} +
+} + +interface SectionContainerProps { + title: string; +} + +export const SectionContainer: React.FC = ({title, children}) => { + const [expanded, setExpanded] = useState(true); + return setExpanded(!expanded)} + title={} + > + {children} + +} + +interface BodySectionProps { + content: any; + encoding?: string; + contentType?: string; +} + +export const BodySection: React.FC = ({content, encoding, contentType}) => { + const MAXIMUM_BYTES_TO_HIGHLIGHT = 10000; // The maximum of chars to highlight in body, in case the response can be megabytes + const supportedLanguages = [['html', 'html'], ['json', 'json'], ['application/grpc', 'json']]; // [[indicator, languageToUse],...] + const jsonLikeFormats = ['json']; + const protobufFormats = ['application/grpc']; + const [isWrapped, setIsWrapped] = useState(false); + + const formatTextBody = (body): string => { + const chunk = body.slice(0, MAXIMUM_BYTES_TO_HIGHLIGHT); + const bodyBuf = encoding === 'base64' ? atob(chunk) : chunk; + + try { + if (jsonLikeFormats.some(format => content?.mimeType?.indexOf(format) > -1)) { + return JSON.stringify(JSON.parse(bodyBuf), null, 2); + } else if (protobufFormats.some(format => content?.mimeType?.indexOf(format) > -1)) { + // Replace all non printable characters (ASCII) + const protobufDecoder = new ProtobufDecoder(bodyBuf, true); + return JSON.stringify(protobufDecoder.decode().toSimple(), null, 2); + } + } catch (error) { + console.error(error); + } + return bodyBuf; + } + + const getLanguage = (mimetype) => { + const chunk = content.text?.slice(0, 100); + if (chunk.indexOf('html') > 0 || chunk.indexOf('HTML') > 0) return supportedLanguages[0][1]; + const language = supportedLanguages.find(el => (mimetype + contentType).indexOf(el[0]) > -1); + return language ? language[1] : 'default'; + } + + return + {content && content.text?.length > 0 && + + + + + +
+ +
setIsWrapped(!isWrapped)}> +
+ {}}/> +
+ Wrap text +
+ + + } + +} + +interface TableSectionProps { + title: string, + arrayToIterate: any[], +} + +export const TableSection: React.FC = ({title, arrayToIterate}) => { + return + { + arrayToIterate && arrayToIterate.length > 0 ? + + + + {arrayToIterate.map(({name, value}, index) => )} + +
+ : + } + +} + +interface HAREntryPolicySectionProps { + service: string, + title: string, + response: any, + latency?: number, + arrayToIterate: any[], +} + + +interface HAREntryPolicySectionCollapsibleTitleProps { + label: string; + matched: string; + isExpanded: boolean; +} + +const HAREntryPolicySectionCollapsibleTitle: React.FC = ({label, matched, isExpanded}) => { + return
+ + {isExpanded ? '-' : '+'} + + + + {label} + {matched} + + +
+} + +interface HAREntryPolicySectionContainerProps { + label: string; + matched: string; + children?: any; +} + +export const HAREntryPolicySectionContainer: React.FC = ({label, matched, children}) => { + const [expanded, setExpanded] = useState(false); + return setExpanded(!expanded)} + title={} + > + {children} + +} + +export const HAREntryTablePolicySection: React.FC = ({service, title, response, latency, arrayToIterate}) => { + return + {arrayToIterate && arrayToIterate.length > 0 ? <> + + + + {arrayToIterate.map(({rule, matched}, index) => { + return (= latency : true)? "Success" : "Failure"}> + <> + {rule.Key && } + {rule.Latency > 0 ? : ''} + {rule.Method && } + {rule.Path && } + {rule.Service && } + {rule.Type && } + {rule.Value && } + + )})} + +
Key:{rule.Key}
Latency:{rule.Latency}
Method: {rule.Method}
Path: {rule.Path}
Service: {service}
Type: {rule.Type}
Value: {rule.Value}
+ + : No rules could be applied to this request.} + +} \ No newline at end of file diff --git a/ui/src/components/EntryDetailed/Kafka/KafkaEntryDetailsContent.tsx b/ui/src/components/EntryDetailed/Kafka/KafkaEntryDetailsContent.tsx new file mode 100644 index 000000000..7fe97954c --- /dev/null +++ b/ui/src/components/EntryDetailed/Kafka/KafkaEntryDetailsContent.tsx @@ -0,0 +1,6 @@ +import React from "react"; + +export const KafkaEntryDetailsContent: React.FC = ({entryData}) => { + + return <>; +} diff --git a/ui/src/components/EntryDetailed/Kafka/KafkaEntryDetailsTitle.tsx b/ui/src/components/EntryDetailed/Kafka/KafkaEntryDetailsTitle.tsx new file mode 100644 index 000000000..4d1aeee2f --- /dev/null +++ b/ui/src/components/EntryDetailed/Kafka/KafkaEntryDetailsTitle.tsx @@ -0,0 +1,6 @@ +import React from "react"; + +export const KafkaEntryDetailsTitle: React.FC = ({entryData}) => { + + return <> +} \ No newline at end of file diff --git a/ui/src/components/EntryDetailed/Rest/RestEntryDetailsContent.tsx b/ui/src/components/EntryDetailed/Rest/RestEntryDetailsContent.tsx new file mode 100644 index 000000000..fe00f15a0 --- /dev/null +++ b/ui/src/components/EntryDetailed/Rest/RestEntryDetailsContent.tsx @@ -0,0 +1,43 @@ +import React, {useState} from "react"; +import styles from "../EntryDetailed.module.sass"; +import Tabs from "../../UI/Tabs"; +import {BodySection, HAREntryTablePolicySection, TableSection} from "../EntrySections"; +import {singleEntryToHAR} from "../../../helpers/utils"; + +const MIME_TYPE_KEY = 'mimeType'; + +export const RestEntryDetailsContent: React.FC = ({entryData}) => { + + const har = singleEntryToHAR(entryData); + const {request, response, timings: {receive}} = har.log.entries[0].entry; + const rulesMatched = har.log.entries[0].rulesMatched + const TABS = [ + {tab: 'request'}, + {tab: 'response'}, + {tab: 'Rules'}, + ]; + + const [currentTab, setCurrentTab] = useState(TABS[0].tab); + + return <> +
+ + {request?.url && {request.url}} +
+ {currentTab === TABS[0].tab && <> + + + {request?.postData && } + + + } + {currentTab === TABS[1].tab && <> + + + + } + {currentTab === TABS[2].tab && <> + + } + ; +} diff --git a/ui/src/components/EntryDetailed/Rest/RestEntryDetailsTitle.tsx b/ui/src/components/EntryDetailed/Rest/RestEntryDetailsTitle.tsx new file mode 100644 index 000000000..3d9925505 --- /dev/null +++ b/ui/src/components/EntryDetailed/Rest/RestEntryDetailsTitle.tsx @@ -0,0 +1,26 @@ +import React from "react"; +import {singleEntryToHAR} from "../../../helpers/utils"; +import StatusCode from "../../UI/StatusCode"; +import {EndpointPath} from "../../UI/EndpointPath"; + +const formatSize = (n: number) => n > 1000 ? `${Math.round(n / 1000)}KB` : `${n} B`; + +export const RestEntryDetailsTitle: React.FC = ({entryData}) => { + + const har = singleEntryToHAR(entryData); + const {log: {entries}} = har; + const {response, request, timings: {receive}} = entries[0].entry; + const {status, statusText, bodySize} = response; + + return har && <> + {status &&
+ +
} +
+ +
+
{formatSize(bodySize)}
+
{status} {statusText}
+
{Math.round(receive)}ms
+ +} \ No newline at end of file diff --git a/ui/src/components/style/HarEntry.module.sass b/ui/src/components/EntryListItem/EntryListItem.module.sass similarity index 56% rename from ui/src/components/style/HarEntry.module.sass rename to ui/src/components/EntryListItem/EntryListItem.module.sass index 5425b1786..23a5421df 100644 --- a/ui/src/components/style/HarEntry.module.sass +++ b/ui/src/components/EntryListItem/EntryListItem.module.sass @@ -1,4 +1,4 @@ -@import 'variables.module' +@import 'src/variables.module' .row display: flex @@ -24,12 +24,40 @@ margin-right: 3px .ruleSuccessRow - border: 1px $success-color solid - border-left: 5px $success-color solid + background: #E8FFF1 + +.ruleSuccessRowSelected + border: 1px #6FCF97 solid + border-left: 5px #6FCF97 solid + margin-left: 10px + margin-right: 3px .ruleFailureRow + background: #FFE9EF + +.ruleFailureRowSelected border: 1px $failure-color solid border-left: 5px $failure-color solid + margin-left: 10px + margin-right: 3px + +.ruleNumberTextFailure + color: #DB2156 + font-family: Source Sans Pro + font-style: normal + font-weight: 600 + font-size: 12px + line-height: 15px + padding-right: 12px + +.ruleNumberTextSuccess + color: #219653 + font-family: Source Sans Pro + font-style: normal + font-weight: 600 + font-size: 12px + line-height: 15px + padding-right: 12px .service text-overflow: ellipsis @@ -45,10 +73,11 @@ .timestamp font-size: 12px color: $secondary-font-color - padding-left: 12px flex-shrink: 0 width: 145px text-align: left + border-left: 1px solid $data-background-color + padding: 6px 0 6px 12px .endpointServiceContainer display: flex @@ -60,6 +89,12 @@ .directionContainer display: flex - border-right: 1px solid $data-background-color - padding: 4px - padding-right: 12px + padding: 4px 12px 4px 4px + +.icon + height: 14px + width: 50px + padding: 5px + background-color: white + border-radius: 15px + box-shadow: 1px 1px 9px -4px black \ No newline at end of file diff --git a/ui/src/components/EntryListItem/EntryListItem.tsx b/ui/src/components/EntryListItem/EntryListItem.tsx new file mode 100644 index 000000000..4a8a8a017 --- /dev/null +++ b/ui/src/components/EntryListItem/EntryListItem.tsx @@ -0,0 +1,85 @@ +import React from "react"; +import styles from './EntryListItem.module.sass'; +import restIcon from '../assets/restIcon.svg'; +import kafkaIcon from '../assets/kafkaIcon.svg'; +import {RestEntry, RestEntryContent} from "./RestEntryContent"; +import {KafkaEntry, KafkaEntryContent} from "./KafkaEntryContent"; + +export interface BaseEntry { + type: string; + timestamp: Date; + id: string; + rules: Rules; + latency: number; +} + +interface Rules { + status: boolean; + latency: number; + numberOfRules: number; +} + +interface EntryProps { + entry: RestEntry | KafkaEntry | any; + setFocusedEntry: (entry: RestEntry | KafkaEntry) => void; + isSelected?: boolean; +} + +export enum EntryType { + Rest = "rest", + Kafka = "kafka" +} + +export const EntryItem: React.FC = ({entry, setFocusedEntry, isSelected}) => { + + let additionalRulesProperties = ""; + let rule = 'latency' in entry.rules + if (rule) { + if (entry.rules.latency !== -1) { + if (entry.rules.latency >= entry.latency) { + additionalRulesProperties = styles.ruleSuccessRow + } else { + additionalRulesProperties = styles.ruleFailureRow + } + if (isSelected) { + additionalRulesProperties += ` ${entry.rules.latency >= entry.latency ? styles.ruleSuccessRowSelected : styles.ruleFailureRowSelected}` + } + } else { + if (entry.rules.status) { + additionalRulesProperties = styles.ruleSuccessRow + } else { + additionalRulesProperties = styles.ruleFailureRow + } + if (isSelected) { + additionalRulesProperties += ` ${entry.rules.status ? styles.ruleSuccessRowSelected : styles.ruleFailureRowSelected}` + } + } + } + + let icon, content; + + switch (entry.type) { + case EntryType.Rest: + content = ; + icon = restIcon; + break; + case EntryType.Kafka: + content = ; + icon = kafkaIcon; + break; + default: + content = ; + icon = restIcon; + break; + } + + return <> +
setFocusedEntry(entry)}> + {icon &&
{icon}
} + {content} +
{new Date(+entry.timestamp)?.toLocaleString()}
+
+ +}; + diff --git a/ui/src/components/EntryListItem/KafkaEntryContent.tsx b/ui/src/components/EntryListItem/KafkaEntryContent.tsx new file mode 100644 index 000000000..b461aef35 --- /dev/null +++ b/ui/src/components/EntryListItem/KafkaEntryContent.tsx @@ -0,0 +1,15 @@ +import {BaseEntry} from "./EntryListItem"; +import React from "react"; + +export interface KafkaEntry extends BaseEntry{ +} + +interface KafkaEntryContentProps { + entry: KafkaEntry; +} + +export const KafkaEntryContent: React.FC = ({entry}) => { + + return <> + +} \ No newline at end of file diff --git a/ui/src/components/EntryListItem/RestEntryContent.tsx b/ui/src/components/EntryListItem/RestEntryContent.tsx new file mode 100644 index 000000000..fb51bff87 --- /dev/null +++ b/ui/src/components/EntryListItem/RestEntryContent.tsx @@ -0,0 +1,82 @@ +import React from "react"; +import StatusCode, {getClassification, StatusCodeClassification} from "../UI/StatusCode"; +import ingoingIconSuccess from "../assets/ingoing-traffic-success.svg"; +import outgoingIconSuccess from "../assets/outgoing-traffic-success.svg"; +import ingoingIconFailure from "../assets/ingoing-traffic-failure.svg"; +import outgoingIconFailure from "../assets/outgoing-traffic-failure.svg"; +import ingoingIconNeutral from "../assets/ingoing-traffic-neutral.svg"; +import outgoingIconNeutral from "../assets/outgoing-traffic-neutral.svg"; +import styles from "./EntryListItem.module.sass"; +import {EndpointPath} from "../UI/EndpointPath"; +import {BaseEntry} from "./EntryListItem"; + +export interface RestEntry extends BaseEntry{ + method?: string, + path: string, + service: string, + statusCode?: number; + url?: string; + isCurrentRevision?: boolean; + isOutgoing?: boolean; +} + +interface RestEntryContentProps { + entry: RestEntry; +} + +export const RestEntryContent: React.FC = ({entry}) => { + const classification = getClassification(entry.statusCode) + const numberOfRules = entry.rules.numberOfRules + + let ingoingIcon; + let outgoingIcon; + switch (classification) { + case StatusCodeClassification.SUCCESS: { + ingoingIcon = ingoingIconSuccess; + outgoingIcon = outgoingIconSuccess; + break; + } + case StatusCodeClassification.FAILURE: { + ingoingIcon = ingoingIconFailure; + outgoingIcon = outgoingIconFailure; + break; + } + case StatusCodeClassification.NEUTRAL: { + ingoingIcon = ingoingIconNeutral; + outgoingIcon = outgoingIconNeutral; + break; + } + } + + let ruleSuccess: boolean; + let rule = 'latency' in entry.rules + if (rule) { + if (entry.rules.latency !== -1) { + ruleSuccess = entry.rules.latency >= entry.latency; + } else { + ruleSuccess = entry.rules.status; + } + } + + return <> + {entry.statusCode &&
+ +
} +
+ +
+ {entry.service} +
+
+ {rule &&
+ {`Rules (${numberOfRules})`} +
} +
+ {entry.isOutgoing ? + outgoing traffic + : + ingoing traffic + } +
+ +} \ No newline at end of file diff --git a/ui/src/components/HarFilters.tsx b/ui/src/components/Filters.tsx similarity index 90% rename from ui/src/components/HarFilters.tsx rename to ui/src/components/Filters.tsx index 5dee2b564..39430a2cb 100644 --- a/ui/src/components/HarFilters.tsx +++ b/ui/src/components/Filters.tsx @@ -1,8 +1,8 @@ import React from "react"; -import styles from './style/HarFilters.module.sass'; -import {HARFilterSelect} from "./HARFilterSelect"; +import styles from './style/Filters.module.sass'; +import {FilterSelect} from "./UI/FilterSelect"; import {TextField} from "@material-ui/core"; -import {ALL_KEY} from "./Select"; +import {ALL_KEY} from "./UI/Select"; interface HarFiltersProps { methodsFilter: Array; @@ -13,7 +13,7 @@ interface HarFiltersProps { setPathFilter: (val: string) => void; } -export const HarFilters: React.FC = ({methodsFilter, setMethodsFilter, statusFilter, setStatusFilter, pathFilter, setPathFilter}) => { +export const Filters: React.FC = ({methodsFilter, setMethodsFilter, statusFilter, setStatusFilter, pathFilter, setPathFilter}) => { return
@@ -59,7 +59,7 @@ const MethodFilter: React.FC = ({methodsFilter, setMethodsFil } return - = ({statusFilter, setS } return - void; - isSelected?: boolean; -} - -export const HarEntry: React.FC = ({entry, setFocusedEntryId, isSelected}) => { - const classification = getClassification(entry.statusCode) - let ingoingIcon; - let outgoingIcon; - switch(classification) { - case StatusCodeClassification.SUCCESS: { - ingoingIcon = ingoingIconSuccess; - outgoingIcon = outgoingIconSuccess; - break; - } - case StatusCodeClassification.FAILURE: { - ingoingIcon = ingoingIconFailure; - outgoingIcon = outgoingIconFailure; - break; - } - case StatusCodeClassification.NEUTRAL: { - ingoingIcon = ingoingIconNeutral; - outgoingIcon = outgoingIconNeutral; - break; - } - } - let backgroundColor = ""; - if ('latency' in entry.rules) { - if (entry.rules.latency !== -1) { - backgroundColor = entry.rules.latency >= entry.latency ? styles.ruleSuccessRow : styles.ruleFailureRow - } else { - backgroundColor = entry.rules.status ? styles.ruleSuccessRow : styles.ruleFailureRow - } - } - return <> -
setFocusedEntryId(entry.id)}> - {entry.statusCode &&
- -
} -
- -
- {entry.service} -
-
-
- {entry.isOutgoing ? - outgoing traffic - : - ingoing traffic - } -
-
{new Date(+entry.timestamp)?.toLocaleString()}
-
- -}; diff --git a/ui/src/components/HarEntryDetailed.tsx b/ui/src/components/HarEntryDetailed.tsx deleted file mode 100644 index 8e86d140f..000000000 --- a/ui/src/components/HarEntryDetailed.tsx +++ /dev/null @@ -1,62 +0,0 @@ -import React from "react"; -import {singleEntryToHAR} from "./utils"; -import styles from './style/HarEntryDetailed.module.sass'; -import HAREntryViewer from "./HarEntryViewer/HAREntryViewer"; -import {makeStyles} from "@material-ui/core"; -import StatusCode from "./StatusCode"; -import {EndpointPath} from "./EndpointPath"; - -const useStyles = makeStyles(() => ({ - entryTitle: { - display: 'flex', - minHeight: 46, - maxHeight: 46, - alignItems: 'center', - marginBottom: 8, - padding: 5, - paddingBottom: 0 - } -})); - -interface HarEntryDetailedProps { - harEntry: any; - classes?: any; -} - -export const formatSize = (n: number) => n > 1000 ? `${Math.round(n / 1000)}KB` : `${n} B`; - -const HarEntryTitle: React.FC = ({har}) => { - const classes = useStyles(); - - const {log: {entries}} = har; - const {response, request, timings: {receive}} = entries[0].entry; - const {status, statusText, bodySize} = response; - - - return
- {status &&
- -
} -
- -
-
{formatSize(bodySize)}
-
{status} {statusText}
-
{Math.round(receive)}ms
-
{'rulesMatched' in entries[0] ? entries[0].rulesMatched?.length : '0'} Rules Applied
-
; -}; - -export const HAREntryDetailed: React.FC = ({classes, harEntry}) => { - const har = singleEntryToHAR(harEntry); - - return <> - {har && } - <> - {har && } - - -}; diff --git a/ui/src/components/HarEntryViewer/HAREntrySections.tsx b/ui/src/components/HarEntryViewer/HAREntrySections.tsx deleted file mode 100644 index 088dd3f51..000000000 --- a/ui/src/components/HarEntryViewer/HAREntrySections.tsx +++ /dev/null @@ -1,266 +0,0 @@ -import styles from "./HAREntrySections.module.sass"; -import React, {useState} from "react"; -import {SyntaxHighlighter} from "../SyntaxHighlighter/index"; -import CollapsibleContainer from "../CollapsibleContainer"; -import FancyTextDisplay from "../FancyTextDisplay"; -import Checkbox from "../Checkbox"; -import ProtobufDecoder from "protobuf-decoder"; -var jp = require('jsonpath'); - -interface HAREntryViewLineProps { - label: string; - value: number | string; -} - -const HAREntryViewLine: React.FC = ({label, value}) => { - return (label && value && - {label} - - - - ) || null; -} - - -interface HAREntrySectionCollapsibleTitleProps { - title: string; - isExpanded: boolean; -} - -const HAREntrySectionCollapsibleTitle: React.FC = ({title, isExpanded}) => { - return
- - {isExpanded ? '-' : '+'} - - {title} -
-} - -interface HAREntrySectionContainerProps { - title: string; -} - -export const HAREntrySectionContainer: React.FC = ({title, children}) => { - const [expanded, setExpanded] = useState(true); - return setExpanded(!expanded)} - title={} - > - {children} - -} - -interface HAREntryBodySectionProps { - content: any; - encoding?: string; - contentType?: string; -} - -export const HAREntryBodySection: React.FC = ({ - content, - encoding, - contentType, - }) => { - const MAXIMUM_BYTES_TO_HIGHLIGHT = 10000; // The maximum of chars to highlight in body, in case the response can be megabytes - const supportedLanguages = [['html', 'html'], ['json', 'json'], ['application/grpc', 'json']]; // [[indicator, languageToUse],...] - const jsonLikeFormats = ['json']; - const protobufFormats = ['application/grpc']; - const [isWrapped, setIsWrapped] = useState(false); - - const formatTextBody = (body): string => { - const chunk = body.slice(0, MAXIMUM_BYTES_TO_HIGHLIGHT); - const bodyBuf = encoding === 'base64' ? atob(chunk) : chunk; - - try { - if (jsonLikeFormats.some(format => content?.mimeType?.indexOf(format) > -1)) { - return JSON.stringify(JSON.parse(bodyBuf), null, 2); - } else if (protobufFormats.some(format => content?.mimeType?.indexOf(format) > -1)) { - // Replace all non printable characters (ASCII) - const protobufDecoder = new ProtobufDecoder(bodyBuf, true); - return JSON.stringify(protobufDecoder.decode().toSimple(), null, 2); - } - } catch (error) { - console.error(error); - } - return bodyBuf; - } - - const getLanguage = (mimetype) => { - const chunk = content.text?.slice(0, 100); - if (chunk.indexOf('html') > 0 || chunk.indexOf('HTML') > 0) return supportedLanguages[0][1]; - const language = supportedLanguages.find(el => (mimetype + contentType).indexOf(el[0]) > -1); - return language ? language[1] : 'default'; - } - - return - {content && content.text?.length > 0 && - - - - - -
- -
setIsWrapped(!isWrapped)}> -
- {}}/> -
- Wrap text -
- - - } - -} - -interface HAREntrySectionProps { - title: string, - arrayToIterate: any[], -} - -export const HAREntryTableSection: React.FC = ({title, arrayToIterate}) => { - return - { - arrayToIterate && arrayToIterate.length > 0 ? - - - - {arrayToIterate.map(({name, value}, index) => )} - -
- : - } - -} - - - -interface HAREntryPolicySectionProps { - service: string, - title: string, - response: any, - latency?: number, - arrayToIterate: any[], -} - - -interface HAREntryPolicySectionCollapsibleTitleProps { - label: string; - matched: string; - isExpanded: boolean; -} - -const HAREntryPolicySectionCollapsibleTitle: React.FC = ({label, matched, isExpanded}) => { - return
- - {isExpanded ? '-' : '+'} - - - - {label} - {matched} - - -
-} - -interface HAREntryPolicySectionContainerProps { - label: string; - matched: string; - children?: any; -} - -export const HAREntryPolicySectionContainer: React.FC = ({label, matched, children}) => { - const [expanded, setExpanded] = useState(false); - return setExpanded(!expanded)} - title={} - > - {children} - -} - -export const HAREntryTablePolicySection: React.FC = ({service, title, response, latency, arrayToIterate}) => { - const base64ToJson = response.content.mimeType === "application/json; charset=utf-8" ? JSON.parse(Buffer.from(response.content.text, "base64").toString()) : {}; - return - { - arrayToIterate && arrayToIterate.length > 0 ? - <> - - - - {arrayToIterate.map(({rule, matched}, index) => { - - - return ( - = latency : true)? "Success" : "Failure"}> - { - - <> - { - rule.Key != "" ? - - : null - } - { - rule.Latency != "" ? - - : null - } - { - rule.Method != "" ? - - : null - } - { - rule.Path != "" ? - - : null - } - { - rule.Service != "" ? - - : null - } - { - rule.Type != "" ? - - : null - } - { - rule.Value != "" ? - - : null - } - - } - - - - ) - } - ) - } - -
Key:{rule.Key}
Latency: {rule.Latency}
Method: {rule.Method}
Path: {rule.Path}
Service: {service}
Type: {rule.Type}
Value: {rule.Value}
- - - : - } - -} \ No newline at end of file diff --git a/ui/src/components/HarEntryViewer/HAREntryViewer.module.sass b/ui/src/components/HarEntryViewer/HAREntryViewer.module.sass deleted file mode 100644 index 1dc344781..000000000 --- a/ui/src/components/HarEntryViewer/HAREntryViewer.module.sass +++ /dev/null @@ -1,60 +0,0 @@ -@import "../style/variables.module" - -.harEntry - font-family: "Source Sans Pro", Lucida Grande, Tahoma, sans-serif - height: 100% - width: 100% - - h3, - h4 - font-family: "Source Sans Pro", Lucida Grande, Tahoma, sans-serif - - .header - background-color: rgb(55, 65, 111) - padding: 0.5rem .75rem .65rem .75rem - border-top-left-radius: 0.25rem - border-top-right-radius: 0.25rem - display: flex - font-size: .75rem - align-items: center - .description - min-width: 25rem - display: flex - align-items: center - justify-content: space-between - .method - padding: 0 .25rem - font-size: 0.75rem - font-weight: bold - border-radius: 0.25rem - border: 0.0625rem solid rgba(255, 255, 255, 0.16) - margin-right: .5rem - > span - margin-left: .5rem - .timing - border-left: 1px solid #627ef7 - margin-left: .3rem - padding-left: .3rem - - .headerClickable - cursor: pointer - &:hover - background: lighten(rgb(55, 65, 111), 10%) - border-top-left-radius: 0 - border-top-right-radius: 0 - - .body - background: $main-background-color - color: $blue-gray - border-radius: 4px - padding: 10px - .bodyHeader - padding: 0 1rem - .endpointURL - font-size: .75rem - display: block - color: $blue-color - text-decoration: none - margin-bottom: .5rem - overflow-wrap: anywhere - padding: 5px 0 \ No newline at end of file diff --git a/ui/src/components/HarEntryViewer/HAREntryViewer.tsx b/ui/src/components/HarEntryViewer/HAREntryViewer.tsx deleted file mode 100644 index e0450e1e7..000000000 --- a/ui/src/components/HarEntryViewer/HAREntryViewer.tsx +++ /dev/null @@ -1,71 +0,0 @@ -import React, {useState} from 'react'; -import styles from './HAREntryViewer.module.sass'; -import Tabs from "../Tabs"; -import {HAREntryTableSection, HAREntryBodySection, HAREntryTablePolicySection} from "./HAREntrySections"; - -const MIME_TYPE_KEY = 'mimeType'; - -const HAREntryDisplay: React.FC = ({har, entry, isCollapsed: initialIsCollapsed, isResponseMocked}) => { - const {request, response, timings: {receive}} = entry; - const rulesMatched = har.log.entries[0].rulesMatched - const TABS = [ - {tab: 'request'}, - { - tab: 'response', - badge: <>{isResponseMocked && MOCK} - }, - { - tab: 'Rules', - }, - ]; - - const [currentTab, setCurrentTab] = useState(TABS[0].tab); - - return
- - {!initialIsCollapsed &&
-
- - {request?.url && {request.url}} -
- { - currentTab === TABS[0].tab && - - - - - {request?.postData && } - - - - } - {currentTab === TABS[1].tab && - - - - - - } - {currentTab === TABS[2].tab && - - } -
} -
; -} - -interface Props { - harObject: any; - className?: string; - isResponseMocked?: boolean; - showTitle?: boolean; -} - -const HAREntryViewer: React.FC = ({harObject, className, isResponseMocked, showTitle=true}) => { - const {log: {entries}} = harObject; - const isCollapsed = entries.length > 1; - return
- {Object.keys(entries).map((entry: any, index) => )} -
-}; - -export default HAREntryViewer; diff --git a/ui/src/components/HarPaging.tsx b/ui/src/components/HarPaging.tsx deleted file mode 100644 index 7fb410bee..000000000 --- a/ui/src/components/HarPaging.tsx +++ /dev/null @@ -1,27 +0,0 @@ -import prevIcon from "./assets/icon-prev.svg"; -import nextIcon from "./assets/icon-next.svg"; -import {Box} from "@material-ui/core"; -import React from "react"; -import styles from './style/HarPaging.module.sass' -import numeral from 'numeral'; - -interface HarPagingProps { - showPageNumber?: boolean; -} - -export const HarPaging: React.FC = ({showPageNumber=false}) => { - - return - { - // harStore.data.moveBack(); todo - }} alt="back"/> - {showPageNumber && - Page - {/*{numeral(harStore.data.currentPage).format(0, 0)}*/} //todo - - } - { - // harStore.data.moveNext(); todo - }} alt="next"/> - -}; \ No newline at end of file diff --git a/ui/src/components/HarPage.tsx b/ui/src/components/TrafficPage.tsx similarity index 66% rename from ui/src/components/HarPage.tsx rename to ui/src/components/TrafficPage.tsx index bfa7dbfb3..8ecfd46f0 100644 --- a/ui/src/components/HarPage.tsx +++ b/ui/src/components/TrafficPage.tsx @@ -1,14 +1,14 @@ import React, {useEffect, useRef, useState} from "react"; -import {HarFilters} from "./HarFilters"; -import {HarEntriesList} from "./HarEntriesList"; +import {Filters} from "./Filters"; +import {EntriesList} from "./EntriesList"; import {makeStyles} from "@material-ui/core"; -import "./style/HarPage.sass"; -import styles from './style/HarEntriesList.module.sass'; -import {HAREntryDetailed} from "./HarEntryDetailed"; +import "./style/TrafficPage.sass"; +import styles from './style/EntriesList.module.sass'; +import {EntryDetailed} from "./EntryDetailed/EntryDetailed"; import playIcon from './assets/run.svg'; import pauseIcon from './assets/pause.svg'; -import variables from './style/variables.module.scss'; -import {StatusBar} from "./StatusBar"; +import variables from '../variables.module.scss'; +import {StatusBar} from "./UI/StatusBar"; import Api, {MizuWebsocketURL} from "../helpers/api"; const useLayoutStyles = makeStyles(() => ({ @@ -43,13 +43,13 @@ interface HarPageProps { const api = new Api(); -export const HarPage: React.FC = ({setAnalyzeStatus, onTLSDetected}) => { +export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLSDetected}) => { const classes = useLayoutStyles(); const [entries, setEntries] = useState([] as any); - const [focusedEntryId, setFocusedEntryId] = useState(null); - const [selectedHarEntry, setSelectedHarEntry] = useState(null); + const [focusedEntry, setFocusedEntry] = useState(null); + const [selectedEntryData, setSelectedEntryData] = useState(null); const [connection, setConnection] = useState(ConnectionStatus.Closed); const [noMoreDataTop, setNoMoreDataTop] = useState(false); const [noMoreDataBottom, setNoMoreDataBottom] = useState(false); @@ -60,8 +60,12 @@ export const HarPage: React.FC = ({setAnalyzeStatus, onTLSDetected const [tappingStatus, setTappingStatus] = useState(null); + const [disableScrollList, setDisableScrollList] = useState(false); + const ws = useRef(null); + const listEntry = useRef(null); + const openWebSocket = () => { ws.current = new WebSocket(MizuWebsocketURL); ws.current.onopen = () => setConnection(ConnectionStatus.Connected); @@ -79,13 +83,18 @@ export const HarPage: React.FC = ({setAnalyzeStatus, onTLSDetected setNoMoreDataBottom(false) return; } - if (!focusedEntryId) setFocusedEntryId(entry.id) + if (!focusedEntry) setFocusedEntry(entry) let newEntries = [...entries]; if (entries.length === 1000) { newEntries = newEntries.splice(1); setNoMoreDataTop(false); } setEntries([...newEntries, entry]) + if(listEntry.current) { + if(isScrollable(listEntry.current.firstChild)) { + setDisableScrollList(true) + } + } break case "status": setTappingStatus(message.tappingStatus); @@ -119,17 +128,17 @@ export const HarPage: React.FC = ({setAnalyzeStatus, onTLSDetected useEffect(() => { - if (!focusedEntryId) return; - setSelectedHarEntry(null); + if (!focusedEntry) return; + setSelectedEntryData(null); (async () => { try { - const entryData = await api.getEntry(focusedEntryId); - setSelectedHarEntry(entryData); + const entryData = await api.getEntry(focusedEntry.id); + setSelectedEntryData(entryData); } catch (error) { console.error(error); } })() - }, [focusedEntryId]) + }, [focusedEntry]) const toggleConnection = () => { setConnection(connection === ConnectionStatus.Connected ? ConnectionStatus.Paused : ConnectionStatus.Connected); @@ -158,6 +167,14 @@ export const HarPage: React.FC = ({setAnalyzeStatus, onTLSDetected } } + const onScrollEvent = (isAtBottom) => { + isAtBottom ? setDisableScrollList(false) : setDisableScrollList(true) + } + + const isScrollable = (element) => { + return element.scrollHeight > element.clientHeight; + }; + return (
@@ -172,32 +189,34 @@ export const HarPage: React.FC = ({setAnalyzeStatus, onTLSDetected
{entries.length > 0 &&
-
-
- {selectedHarEntry && - } + {selectedEntryData && }
} {tappingStatus?.pods != null && } diff --git a/ui/src/components/Checkbox.tsx b/ui/src/components/UI/Checkbox.tsx similarity index 100% rename from ui/src/components/Checkbox.tsx rename to ui/src/components/UI/Checkbox.tsx diff --git a/ui/src/components/CollapsibleContainer.tsx b/ui/src/components/UI/CollapsibleContainer.tsx similarity index 95% rename from ui/src/components/CollapsibleContainer.tsx rename to ui/src/components/UI/CollapsibleContainer.tsx index aad6b1552..4c0452623 100644 --- a/ui/src/components/CollapsibleContainer.tsx +++ b/ui/src/components/UI/CollapsibleContainer.tsx @@ -1,6 +1,6 @@ import React, {useState} from "react"; -import collapsedImg from "./assets/collapsed.svg"; -import expandedImg from "./assets/expanded.svg"; +import collapsedImg from "../assets/collapsed.svg"; +import expandedImg from "../assets/expanded.svg"; import "./style/CollapsibleContainer.sass"; interface Props { diff --git a/ui/src/components/EndpointPath.tsx b/ui/src/components/UI/EndpointPath.tsx similarity index 100% rename from ui/src/components/EndpointPath.tsx rename to ui/src/components/UI/EndpointPath.tsx diff --git a/ui/src/components/FancyTextDisplay.tsx b/ui/src/components/UI/FancyTextDisplay.tsx similarity index 97% rename from ui/src/components/FancyTextDisplay.tsx rename to ui/src/components/UI/FancyTextDisplay.tsx index c61a85bd5..91f10f4bf 100644 --- a/ui/src/components/FancyTextDisplay.tsx +++ b/ui/src/components/UI/FancyTextDisplay.tsx @@ -1,6 +1,6 @@ import React, { useEffect, useState } from 'react'; import { CopyToClipboard } from 'react-copy-to-clipboard'; -import duplicateImg from "./assets/duplicate.svg"; +import duplicateImg from "../assets/duplicate.svg"; import './style/FancyTextDisplay.sass'; interface Props { diff --git a/ui/src/components/HARFilterSelect.tsx b/ui/src/components/UI/FilterSelect.tsx similarity index 79% rename from ui/src/components/HARFilterSelect.tsx rename to ui/src/components/UI/FilterSelect.tsx index c4bc51804..a2247b6d8 100644 --- a/ui/src/components/HARFilterSelect.tsx +++ b/ui/src/components/UI/FilterSelect.tsx @@ -1,6 +1,6 @@ import React from "react"; import { MenuItem } from '@material-ui/core'; -import style from './style/HARFilterSelect.module.sass'; +import style from './style/FilterSelect.module.sass'; import { Select, SelectProps } from "./Select"; interface HARFilterSelectProps extends SelectProps { @@ -12,7 +12,7 @@ interface HARFilterSelectProps extends SelectProps { transformDisplay?: (string) => string; } -export const HARFilterSelect: React.FC = ({items, value, onChange, label, allowMultiple= false, transformDisplay}) => { +export const FilterSelect: React.FC = ({items, value, onChange, label, allowMultiple= false, transformDisplay}) => { return