diff --git a/.github/workflows/pr_validation.yml b/.github/workflows/pr_validation.yml index 4c1bf53fd..dd196ec18 100644 --- a/.github/workflows/pr_validation.yml +++ b/.github/workflows/pr_validation.yml @@ -44,3 +44,18 @@ jobs: - name: Build Agent run: make agent + + build-ui: + name: Build UI + runs-on: ubuntu-latest + steps: + - name: Set up Node 14 + uses: actions/setup-node@v2 + with: + node-version: '14' + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Build UI + run: make ui diff --git a/README.md b/README.md index aa50a5bbb..b071f6390 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,17 @@ ![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg) +

+ + GitHub Latest Release + + + GitHub License + + + Slack + +

+ # The API Traffic Viewer for Kubernetes A simple-yet-powerful API traffic viewer for Kubernetes enabling you to view all API communication between microservices to help your debug and troubleshoot regressions. diff --git a/acceptanceTests/cypress.json b/acceptanceTests/cypress.json index 48150bbb3..a0fe7f827 100644 --- a/acceptanceTests/cypress.json +++ b/acceptanceTests/cypress.json @@ -4,11 +4,17 @@ "viewportHeight": 1080, "video": false, "screenshotOnRunFailure": false, + "testFiles": ["tests/GuiPort.js", "tests/MultipleNamespaces.js", + "tests/Redact.js", + "tests/NoRedact.js", "tests/Regex.js"], + "env": { - "testUrl": "http://localhost:8899/" + "testUrl": "http://localhost:8899/", + "redactHeaderContent": "User-Header[REDACTED]", + "redactBodyContent": "{ \"User\": \"[REDACTED]\" }" } } diff --git a/acceptanceTests/cypress/integration/page_objects/StatusBar.js b/acceptanceTests/cypress/integration/testHelpers/StatusBarHelper.js similarity index 100% rename from acceptanceTests/cypress/integration/page_objects/StatusBar.js rename to acceptanceTests/cypress/integration/testHelpers/StatusBarHelper.js diff --git a/acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js b/acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js new file mode 100644 index 000000000..e90ec7e11 --- /dev/null +++ b/acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js @@ -0,0 +1,9 @@ +export function isValueExistsInElement(shouldInclude, content, domPathToContainer){ + it(`should ${shouldInclude ? '' : 'not'} include '${content}'`, function () { + cy.get(domPathToContainer).then(htmlText => { + const allTextString = htmlText.text(); + if (allTextString.includes(content) !== shouldInclude) + throw new Error(`One of the containers part contains ${content}`) + }); + }); +} diff --git a/acceptanceTests/cypress/integration/tests/MultipleNamespaces.js b/acceptanceTests/cypress/integration/tests/MultipleNamespaces.js index 110f9d13f..96b032e10 100644 --- a/acceptanceTests/cypress/integration/tests/MultipleNamespaces.js +++ b/acceptanceTests/cypress/integration/tests/MultipleNamespaces.js @@ -1,4 +1,4 @@ -import {findLineAndCheck, getExpectedDetailsDict} from '../page_objects/StatusBar'; +import {findLineAndCheck, getExpectedDetailsDict} from '../testHelpers/StatusBarHelper'; it('opening', function () { cy.visit(Cypress.env('testUrl')); diff --git a/acceptanceTests/cypress/integration/tests/NoRedact.js b/acceptanceTests/cypress/integration/tests/NoRedact.js new file mode 100644 index 000000000..c76c5c65d --- /dev/null +++ b/acceptanceTests/cypress/integration/tests/NoRedact.js @@ -0,0 +1,8 @@ +import {isValueExistsInElement} from '../testHelpers/TrafficHelper'; + +it('Loading Mizu', function () { + cy.visit(Cypress.env('testUrl')); +}) + +isValueExistsInElement(false, Cypress.env('redactHeaderContent'), '#tbody-Headers'); +isValueExistsInElement(false, Cypress.env('redactBodyContent'), '.hljs'); diff --git a/acceptanceTests/cypress/integration/tests/Redact.js b/acceptanceTests/cypress/integration/tests/Redact.js new file mode 100644 index 000000000..293ebfd14 --- /dev/null +++ b/acceptanceTests/cypress/integration/tests/Redact.js @@ -0,0 +1,8 @@ +import {isValueExistsInElement} from '../testHelpers/TrafficHelper'; + +it('Loading Mizu', function () { + cy.visit(Cypress.env('testUrl')); +}) + +isValueExistsInElement(true, Cypress.env('redactHeaderContent'), '#tbody-Headers'); +isValueExistsInElement(true, Cypress.env('redactBodyContent'), '.hljs'); diff --git a/acceptanceTests/cypress/integration/tests/Regex.js b/acceptanceTests/cypress/integration/tests/Regex.js index b11ef2e58..de449a7a4 100644 --- a/acceptanceTests/cypress/integration/tests/Regex.js +++ b/acceptanceTests/cypress/integration/tests/Regex.js @@ -1,4 +1,4 @@ -import {getExpectedDetailsDict, checkLine} from '../page_objects/StatusBar'; +import {getExpectedDetailsDict, checkLine} from '../testHelpers/StatusBarHelper'; it('opening', function () { diff --git a/acceptanceTests/tap_test.go b/acceptanceTests/tap_test.go index d8f309d11..4a61db1ee 100644 --- a/acceptanceTests/tap_test.go +++ b/acceptanceTests/tap_test.go @@ -3,7 +3,6 @@ package acceptanceTests import ( "archive/zip" "bytes" - "encoding/json" "fmt" "io/ioutil" "net/http" @@ -378,59 +377,7 @@ func TestTapRedact(t *testing.T) { } } - redactCheckFunc := func() error { - timestamp := time.Now().UnixNano() / int64(time.Millisecond) - - entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second) - if err != nil { - return err - } - err = checkEntriesAtLeast(entries, 1) - if err != nil { - return err - } - firstEntry := entries[0] - - entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"]) - requestResult, requestErr := executeHttpGetRequest(entryUrl) - if requestErr != nil { - return fmt.Errorf("failed to get entry, err: %v", requestErr) - } - - entry := requestResult.(map[string]interface{})["data"].(map[string]interface{}) - request := entry["request"].(map[string]interface{}) - - headers := request["_headers"].([]interface{}) - for _, headerInterface := range headers { - header := headerInterface.(map[string]interface{}) - if header["name"].(string) != "User-Header" { - continue - } - - userHeader := header["value"].(string) - if userHeader != "[REDACTED]" { - return fmt.Errorf("unexpected result - user agent is not redacted") - } - } - - postData := request["postData"].(map[string]interface{}) - textDataStr := postData["text"].(string) - - var textData map[string]string - if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil { - return fmt.Errorf("failed to parse text data, err: %v", parseErr) - } - - if textData["User"] != "[REDACTED]" { - return fmt.Errorf("unexpected result - user in body is not redacted") - } - - return nil - } - if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil { - t.Errorf("%v", err) - return - } + runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/Redact.js\"")) } func TestTapNoRedact(t *testing.T) { @@ -482,59 +429,7 @@ func TestTapNoRedact(t *testing.T) { } } - redactCheckFunc := func() error { - timestamp := time.Now().UnixNano() / int64(time.Millisecond) - - entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second) - if err != nil { - return err - } - err = checkEntriesAtLeast(entries, 1) - if err != nil { - return err - } - firstEntry := entries[0] - - entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"]) - requestResult, requestErr := executeHttpGetRequest(entryUrl) - if requestErr != nil { - return fmt.Errorf("failed to get entry, err: %v", requestErr) - } - - entry := requestResult.(map[string]interface{})["data"].(map[string]interface{}) - request := entry["request"].(map[string]interface{}) - - headers := request["_headers"].([]interface{}) - for _, headerInterface := range headers { - header := headerInterface.(map[string]interface{}) - if header["name"].(string) != "User-Header" { - continue - } - - userHeader := header["value"].(string) - if userHeader == "[REDACTED]" { - return fmt.Errorf("unexpected result - user agent is redacted") - } - } - - postData := request["postData"].(map[string]interface{}) - textDataStr := postData["text"].(string) - - var textData map[string]string - if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil { - return fmt.Errorf("failed to parse text data, err: %v", parseErr) - } - - if textData["User"] == "[REDACTED]" { - return fmt.Errorf("unexpected result - user in body is redacted") - } - - return nil - } - if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil { - t.Errorf("%v", err) - return - } + runCypressTests(t, "npx cypress run --spec \"cypress/integration/tests/NoRedact.js\"") } func TestTapRegexMasking(t *testing.T) { diff --git a/agent/main.go b/agent/main.go index 359caa272..33e7eaa27 100644 --- a/agent/main.go +++ b/agent/main.go @@ -128,7 +128,7 @@ func main() { syncEntriesConfig := getSyncEntriesConfig() if syncEntriesConfig != nil { if err := up9.SyncEntries(syncEntriesConfig); err != nil { - panic(fmt.Sprintf("Error syncing entries, err: %v", err)) + logger.Log.Error("Error syncing entries, err: %v", err) } } diff --git a/cli/README.md.TEMPLATE b/cli/README.md.TEMPLATE index ed922a8c8..cc6960e16 100644 --- a/cli/README.md.TEMPLATE +++ b/cli/README.md.TEMPLATE @@ -1,6 +1,7 @@ # Mizu release _SEM_VER_ +Full changelog for stable release see in [docs](https://github.com/up9inc/mizu/blob/main/docs/CHANGELOG.md) -Download Mizu for your platform +## Download Mizu for your platform **Mac** (Intel) ``` diff --git a/cli/cmd/installRunner.go b/cli/cmd/installRunner.go index b4912e7ce..2802bc011 100644 --- a/cli/cmd/installRunner.go +++ b/cli/cmd/installRunner.go @@ -101,7 +101,8 @@ func watchApiServerPodReady(ctx context.Context, kubernetesProvider *kubernetes. podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex) eventChan, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper) - timeAfter := time.After(1 * time.Minute) + apiServerTimeoutSec := config.GetIntEnvConfig(config.ApiServerTimeoutSec, 120) + timeAfter := time.After(time.Duration(apiServerTimeoutSec) * time.Second) for { select { case wEvent, ok := <-eventChan: diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 3210acc9a..fec222338 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -306,7 +306,9 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex) eventChan, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper) isPodReady := false - timeAfter := time.After(25 * time.Second) + + apiServerTimeoutSec := config.GetIntEnvConfig(config.ApiServerTimeoutSec, 120) + timeAfter := time.After(time.Duration(apiServerTimeoutSec) * time.Second) for { select { case wEvent, ok := <-eventChan: diff --git a/cli/config/envConfig.go b/cli/config/envConfig.go index 78f91e00b..acbe5e1ab 100644 --- a/cli/config/envConfig.go +++ b/cli/config/envConfig.go @@ -7,6 +7,7 @@ import ( const ( ApiServerRetries = "API_SERVER_RETRIES" + ApiServerTimeoutSec = "API_SERVER_TIMEOUT_SEC" ) func GetIntEnvConfig(key string, defaultValue int) int { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md new file mode 100644 index 000000000..f56ad4038 --- /dev/null +++ b/docs/CHANGELOG.md @@ -0,0 +1,43 @@ +# CHANGELOG +This document summarizes main and fixes changes published in stable (aka `main`) branch of this project. +Ongoing work and development releases are under `develop` branch. + +## 0.22.0 + +### main features +* Service Mesh support -- mizu is now capable to tap mTLS traffic between pods connected by Istio service mesh + * Use `--service-mesh` option to enable this feature +* New installation option - have the same Mizu functionality as long living pods in your cluster, with password protection + * To install use `mizu install` command + * To access use `mizu view` or `kubectl -n mizu port-forward svc/mizu-api-server` + * To uninstall run `mizu clean` +* At first login + * Set admin password as prompted, use it to login to mizu later on. + * After login, user should select cluster namespaces to tap: by default all namespaces in the cluster are selected, user can select/unselect according to their needs. These settings are retained and can be modified at any time via Settings menu (cog icon on the top-right) + + +### improvements +* improved Mizu permissions/roles logic to support clusters with strict PodSecurityPolicy (PSP) -- see [PERMISSIONS](PERMISSIONS.md) doc for more details + +### notable bug fixes +* mizu now works properly when API service is exposed via HTTPS url +* mizu now properly displays KAFKA message body + + + + +## 0.21.0 + +### main features +* New traffic search & stream exprience +* Rich query language with full-text search capabilities on headers & body +* Distinct live-streaming vs paging/browsing modes, all with filter applied + +### improvements +* GUI - source and destination IP addresses & service names for each traffic item +* GUI - Mizu health - display warning sign in top bar when not all requested pods are successfully tapped +* GUI - pod tapping status reflected in the list (ok or problem) +* Mizu telemetry - report platform type + +### fixes +* Request duration and body size properly shown in GUI (instead of -1) diff --git a/tap/extensions/http/helpers.go b/tap/extensions/http/helpers.go index 9348ea3cd..9bf15fca1 100644 --- a/tap/extensions/http/helpers.go +++ b/tap/extensions/http/helpers.go @@ -3,14 +3,15 @@ package main import ( "encoding/json" "fmt" + "strconv" "github.com/up9inc/mizu/tap/api" ) func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}) { newMap = make(map[string]interface{}) - for _, header := range mapSlice { - h := header.(map[string]interface{}) + for _, item := range mapSlice { + h := item.(map[string]interface{}) newMap[h["name"].(string)] = h["value"] } @@ -19,8 +20,8 @@ func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{} func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (representation string) { var table []api.TableData - for _, header := range mapSlice { - h := header.(map[string]interface{}) + for _, item := range mapSlice { + h := item.(map[string]interface{}) selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, h["name"].(string)) table = append(table, api.TableData{ Name: h["name"].(string), @@ -33,3 +34,19 @@ func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (re representation = string(obj) return } + +func representSliceAsTable(slice []interface{}, selectorPrefix string) (representation string) { + var table []api.TableData + for i, item := range slice { + selector := fmt.Sprintf("%s[%d]", selectorPrefix, i) + table = append(table, api.TableData{ + Name: strconv.Itoa(i), + Value: item.(interface{}), + Selector: selector, + }) + } + + obj, _ := json.Marshal(table) + representation = string(obj) + return +} diff --git a/tap/extensions/http/main.go b/tap/extensions/http/main.go index b443867de..cf6a905f5 100644 --- a/tap/extensions/http/main.go +++ b/tap/extensions/http/main.go @@ -9,6 +9,7 @@ import ( "log" "net/http" "net/url" + "strings" "time" "github.com/up9inc/mizu/tap/api" @@ -209,6 +210,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, request["url"] = reqDetails["url"].(string) reqDetails["targetUri"] = reqDetails["url"] reqDetails["path"] = path + reqDetails["pathSegments"] = strings.Split(path, "/")[1:] reqDetails["summary"] = path // Rearrange the maps for the querying @@ -296,6 +298,15 @@ func representRequest(request map[string]interface{}) (repRequest []interface{}) Data: string(details), }) + pathSegments := request["pathSegments"].([]interface{}) + if len(pathSegments) > 1 { + repRequest = append(repRequest, api.SectionData{ + Type: api.TABLE, + Title: "Path Segments", + Data: representSliceAsTable(pathSegments, `request.pathSegments`), + }) + } + repRequest = append(repRequest, api.SectionData{ Type: api.TABLE, Title: "Headers", diff --git a/tap/extensions/kafka/go.mod b/tap/extensions/kafka/go.mod index 113627f94..cdb75609a 100644 --- a/tap/extensions/kafka/go.mod +++ b/tap/extensions/kafka/go.mod @@ -3,6 +3,8 @@ module github.com/up9inc/mizu/tap/extensions/kafka go 1.16 require ( + github.com/fatih/camelcase v1.0.0 + github.com/ohler55/ojg v1.12.12 github.com/segmentio/kafka-go v0.4.17 github.com/up9inc/mizu/tap/api v0.0.0 ) diff --git a/tap/extensions/kafka/go.sum b/tap/extensions/kafka/go.sum index 70ec03308..e3d62474b 100644 --- a/tap/extensions/kafka/go.sum +++ b/tap/extensions/kafka/go.sum @@ -1,6 +1,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= @@ -16,6 +18,8 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/ohler55/ojg v1.12.12 h1:hepbQFn7GHAecTPmwS3j5dCiOLsOpzPLvhiqnlAVAoE= +github.com/ohler55/ojg v1.12.12/go.mod h1:LBbIVRAgoFbYBXQhRhuEpaJIqq+goSO63/FQ+nyJU88= github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/tap/extensions/kafka/helpers.go b/tap/extensions/kafka/helpers.go index dffe30bd8..68264c077 100644 --- a/tap/extensions/kafka/helpers.go +++ b/tap/extensions/kafka/helpers.go @@ -3,8 +3,13 @@ package main import ( "encoding/json" "fmt" + "reflect" "strconv" + "strings" + "github.com/fatih/camelcase" + "github.com/ohler55/ojg/jp" + "github.com/ohler55/ojg/oj" "github.com/up9inc/mizu/tap/api" ) @@ -289,17 +294,12 @@ func representProduceRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) payload := data["payload"].(map[string]interface{}) - topicData := "" - _topicData := payload["topicData"] - if _topicData != nil { - x, _ := json.Marshal(_topicData.([]interface{})) - topicData = string(x) - } + topicData := payload["topicData"] transactionalID := "" if payload["transactionalID"] != nil { transactionalID = payload["transactionalID"].(string) } - repPayload, _ := json.Marshal([]api.TableData{ + repTransactionDetails, _ := json.Marshal([]api.TableData{ { Name: "Transactional ID", Value: transactionalID, @@ -315,18 +315,73 @@ func representProduceRequest(data map[string]interface{}) []interface{} { Value: fmt.Sprintf("%d", int(payload["timeout"].(float64))), Selector: `request.payload.timeout`, }, - { - Name: "Topic Data", - Value: topicData, - Selector: `request.payload.topicData`, - }, }) rep = append(rep, api.SectionData{ Type: api.TABLE, - Title: "Payload", - Data: string(repPayload), + Title: "Transaction Details", + Data: string(repTransactionDetails), }) + if topicData != nil { + for _, _topic := range topicData.([]interface{}) { + topic := _topic.(map[string]interface{}) + topicName := topic["topic"].(string) + partitions := topic["partitions"].(map[string]interface{}) + partitionsJson, err := json.Marshal(partitions) + if err != nil { + return rep + } + + repPartitions, _ := json.Marshal([]api.TableData{ + { + Name: "Length", + Value: partitions["length"], + Selector: `request.payload.transactionalID`, + }, + }) + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: fmt.Sprintf("Partitions (topic: %s)", topicName), + Data: string(repPartitions), + }) + + obj, err := oj.ParseString(string(partitionsJson)) + recordBatchPath, err := jp.ParseString(`partitionData.records.recordBatch`) + recordBatchresults := recordBatchPath.Get(obj) + if len(recordBatchresults) > 0 { + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: fmt.Sprintf("Record Batch (topic: %s)", topicName), + Data: representMapAsTable(recordBatchresults[0].(map[string]interface{}), `request.payload.topicData.partitions.partitionData.records.recordBatch`, []string{"record"}), + }) + } + + recordsPath, err := jp.ParseString(`partitionData.records.recordBatch.record`) + recordsResults := recordsPath.Get(obj) + if len(recordsResults) > 0 { + records := recordsResults[0].([]interface{}) + for i, _record := range records { + record := _record.(map[string]interface{}) + value := record["value"] + delete(record, "value") + + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: fmt.Sprintf("Record [%d] Details (topic: %s)", i, topicName), + Data: representMapAsTable(record, fmt.Sprintf(`request.payload.topicData.partitions.partitionData.records.recordBatch.record[%d]`, i), []string{"value"}), + }) + + rep = append(rep, api.SectionData{ + Type: api.BODY, + Title: fmt.Sprintf("Record [%d] Value", i), + Data: value.(string), + Selector: fmt.Sprintf(`request.payload.topicData.partitions.partitionData.records.recordBatch.record[%d].value`, i), + }) + } + } + } + } + return rep } @@ -336,21 +391,12 @@ func representProduceResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) payload := data["payload"].(map[string]interface{}) - responses := "" - if payload["responses"] != nil { - _responses, _ := json.Marshal(payload["responses"].([]interface{})) - responses = string(_responses) - } + responses := payload["responses"] throttleTimeMs := "" if payload["throttleTimeMs"] != nil { throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) } repPayload, _ := json.Marshal([]api.TableData{ - { - Name: "Responses", - Value: string(responses), - Selector: `response.payload.responses`, - }, { Name: "Throttle Time (ms)", Value: throttleTimeMs, @@ -359,10 +405,31 @@ func representProduceResponse(data map[string]interface{}) []interface{} { }) rep = append(rep, api.SectionData{ Type: api.TABLE, - Title: "Payload", + Title: "Transaction Details", Data: string(repPayload), }) + if responses != nil { + for i, _response := range responses.([]interface{}) { + response := _response.(map[string]interface{}) + + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: fmt.Sprintf("Response [%d]", i), + Data: representMapAsTable(response, fmt.Sprintf(`response.payload.responses[%d]`, i), []string{"partitionResponses"}), + }) + + for j, _partitionResponse := range response["partitionResponses"].([]interface{}) { + partitionResponse := _partitionResponse.(map[string]interface{}) + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: fmt.Sprintf("Response [%d] Partition Response [%d]", i, j), + Data: representMapAsTable(partitionResponse, fmt.Sprintf(`response.payload.responses[%d].partitionResponses[%d]`, i, j), []string{}), + }) + } + } + } + return rep } @@ -372,11 +439,7 @@ func representFetchRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) payload := data["payload"].(map[string]interface{}) - topics := "" - if payload["topics"] != nil { - _topics, _ := json.Marshal(payload["topics"].([]interface{})) - topics = string(_topics) - } + topics := payload["topics"] replicaId := "" if payload["replicaId"] != nil { replicaId = fmt.Sprintf("%d", int(payload["replicaId"].(float64))) @@ -442,11 +505,6 @@ func representFetchRequest(data map[string]interface{}) []interface{} { Value: sessionEpoch, Selector: `request.payload.sessionEpoch`, }, - { - Name: "Topics", - Value: topics, - Selector: `request.payload.topics`, - }, { Name: "Forgotten Topics Data", Value: forgottenTopicsData, @@ -460,10 +518,26 @@ func representFetchRequest(data map[string]interface{}) []interface{} { }) rep = append(rep, api.SectionData{ Type: api.TABLE, - Title: "Payload", + Title: "Transaction Details", Data: string(repPayload), }) + if topics != nil { + for i, _topic := range topics.([]interface{}) { + topic := _topic.(map[string]interface{}) + topicName := topic["topic"].(string) + for j, _partition := range topic["partitions"].([]interface{}) { + partition := _partition.(map[string]interface{}) + + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: fmt.Sprintf("Partition [%d] (topic: %s)", j, topicName), + Data: representMapAsTable(partition, fmt.Sprintf(`request.payload.topics[%d].partitions[%d]`, i, j), []string{}), + }) + } + } + } + return rep } @@ -473,11 +547,7 @@ func representFetchResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) payload := data["payload"].(map[string]interface{}) - responses := "" - if payload["responses"] != nil { - _responses, _ := json.Marshal(payload["responses"].([]interface{})) - responses = string(_responses) - } + responses := payload["responses"] throttleTimeMs := "" if payload["throttleTimeMs"] != nil { throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) @@ -506,18 +576,56 @@ func representFetchResponse(data map[string]interface{}) []interface{} { Value: sessionId, Selector: `response.payload.sessionId`, }, - { - Name: "Responses", - Value: responses, - Selector: `response.payload.responses`, - }, }) rep = append(rep, api.SectionData{ Type: api.TABLE, - Title: "Payload", + Title: "Transaction Details", Data: string(repPayload), }) + if responses != nil { + for i, _response := range responses.([]interface{}) { + response := _response.(map[string]interface{}) + topicName := response["topic"].(string) + + for j, _partitionResponse := range response["partitionResponses"].([]interface{}) { + partitionResponse := _partitionResponse.(map[string]interface{}) + recordSet := partitionResponse["recordSet"].(map[string]interface{}) + + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: fmt.Sprintf("Response [%d] Partition Response [%d] (topic: %s)", i, j, topicName), + Data: representMapAsTable(partitionResponse, fmt.Sprintf(`response.payload.responses[%d].partitionResponses[%d]`, i, j), []string{"recordSet"}), + }) + + recordBatch := recordSet["recordBatch"].(map[string]interface{}) + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: fmt.Sprintf("Response [%d] Partition Response [%d] Record Batch (topic: %s)", i, j, topicName), + Data: representMapAsTable(recordBatch, fmt.Sprintf(`response.payload.responses[%d].partitionResponses[%d].recordSet.recordBatch`, i, j), []string{"record"}), + }) + + for k, _record := range recordBatch["record"].([]interface{}) { + record := _record.(map[string]interface{}) + value := record["value"] + + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: fmt.Sprintf("Response [%d] Partition Response [%d] Record [%d] (topic: %s)", i, j, k, topicName), + Data: representMapAsTable(record, fmt.Sprintf(`response.payload.responses[%d].partitionResponses[%d].recordSet.recordBatch.record[%d]`, i, j, k), []string{"value"}), + }) + + rep = append(rep, api.SectionData{ + Type: api.BODY, + Title: fmt.Sprintf("Response [%d] Partition Response [%d] Record [%d] Value (topic: %s)", i, j, k, topicName), + Data: value.(string), + Selector: fmt.Sprintf(`response.payload.responses[%d].partitionResponses[%d].recordSet.recordBatch.record[%d].value`, i, j, k), + }) + } + } + } + } + return rep } @@ -591,17 +699,11 @@ func representCreateTopicsRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) payload := data["payload"].(map[string]interface{}) - topics, _ := json.Marshal(payload["topics"].([]interface{})) validateOnly := "" if payload["validateOnly"] != nil { validateOnly = strconv.FormatBool(payload["validateOnly"].(bool)) } repPayload, _ := json.Marshal([]api.TableData{ - { - Name: "Topics", - Value: string(topics), - Selector: `request.payload.topics`, - }, { Name: "Timeout (ms)", Value: fmt.Sprintf("%d", int(payload["timeoutMs"].(float64))), @@ -615,10 +717,20 @@ func representCreateTopicsRequest(data map[string]interface{}) []interface{} { }) rep = append(rep, api.SectionData{ Type: api.TABLE, - Title: "Payload", + Title: "Transaction Details", Data: string(repPayload), }) + for i, _topic := range payload["topics"].([]interface{}) { + topic := _topic.(map[string]interface{}) + + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: fmt.Sprintf("Topic [%d]", i), + Data: representMapAsTable(topic, fmt.Sprintf(`request.payload.topics[%d]`, i), []string{}), + }) + } + return rep } @@ -628,7 +740,6 @@ func representCreateTopicsResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) payload := data["payload"].(map[string]interface{}) - topics, _ := json.Marshal(payload["topics"].([]interface{})) throttleTimeMs := "" if payload["throttleTimeMs"] != nil { throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) @@ -639,18 +750,23 @@ func representCreateTopicsResponse(data map[string]interface{}) []interface{} { Value: throttleTimeMs, Selector: `response.payload.throttleTimeMs`, }, - { - Name: "Topics", - Value: string(topics), - Selector: `response.payload.topics`, - }, }) rep = append(rep, api.SectionData{ Type: api.TABLE, - Title: "Payload", + Title: "Transaction Details", Data: string(repPayload), }) + for i, _topic := range payload["topics"].([]interface{}) { + topic := _topic.(map[string]interface{}) + + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: fmt.Sprintf("Topic [%d]", i), + Data: representMapAsTable(topic, fmt.Sprintf(`response.payload.topics[%d]`, i), []string{}), + }) + } + return rep } @@ -727,3 +843,42 @@ func representDeleteTopicsResponse(data map[string]interface{}) []interface{} { return rep } + +func contains(s []string, str string) bool { + for _, v := range s { + if v == str { + return true + } + } + + return false +} + +func representMapAsTable(mapData map[string]interface{}, selectorPrefix string, ignoreKeys []string) (representation string) { + var table []api.TableData + for key, value := range mapData { + if contains(ignoreKeys, key) { + continue + } + switch reflect.ValueOf(value).Kind() { + case reflect.Map: + fallthrough + case reflect.Slice: + x, err := json.Marshal(value) + value = string(x) + if err != nil { + continue + } + } + selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, key) + table = append(table, api.TableData{ + Name: strings.Join(camelcase.Split(strings.Title(key)), " "), + Value: value, + Selector: selector, + }) + } + + obj, _ := json.Marshal(table) + representation = string(obj) + return +} diff --git a/tap/extensions/redis/helpers.go b/tap/extensions/redis/helpers.go index 3f8b61791..1661d1e0b 100644 --- a/tap/extensions/redis/helpers.go +++ b/tap/extensions/redis/helpers.go @@ -42,11 +42,6 @@ func representGeneric(generic map[string]interface{}, selectorPrefix string) (re Value: generic["key"].(string), Selector: fmt.Sprintf("%skey", selectorPrefix), }, - { - Name: "Value", - Value: generic["value"].(string), - Selector: fmt.Sprintf("%svalue", selectorPrefix), - }, { Name: "Keyword", Value: generic["keyword"].(string), @@ -59,5 +54,12 @@ func representGeneric(generic map[string]interface{}, selectorPrefix string) (re Data: string(details), }) + representation = append(representation, api.SectionData{ + Type: api.BODY, + Title: "Value", + Data: generic["value"].(string), + Selector: fmt.Sprintf("%svalue", selectorPrefix), + }) + return } diff --git a/ui/package-lock.json b/ui/package-lock.json index 98bfc9969..7dbac7c07 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -7856,6 +7856,11 @@ "pify": "^4.0.1" } }, + "hamt_plus": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/hamt_plus/-/hamt_plus-1.0.2.tgz", + "integrity": "sha1-4hwlKWjH4zsg9qGwlM2FeHomVgE=" + }, "handle-thing": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz", @@ -14229,6 +14234,14 @@ "picomatch": "^2.2.1" } }, + "recoil": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/recoil/-/recoil-0.5.2.tgz", + "integrity": "sha512-Edibzpu3dbUMLy6QRg73WL8dvMl9Xqhp+kU+f2sJtXxsaXvAlxU/GcnDE8HXPkprXrhHF2e6SZozptNvjNF5fw==", + "requires": { + "hamt_plus": "1.0.2" + } + }, "recursive-readdir": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.2.tgz", diff --git a/ui/package.json b/ui/package.json index 64763c557..a3de9574b 100644 --- a/ui/package.json +++ b/ui/package.json @@ -31,6 +31,7 @@ "react-scrollable-feed-virtualized": "^1.4.9", "react-syntax-highlighter": "^15.4.3", "react-toastify": "^8.0.3", + "recoil": "^0.5.2", "typescript": "^4.2.4", "web-vitals": "^1.1.1", "xml-formatter": "^2.6.0" diff --git a/ui/src/EntApp.tsx b/ui/src/EntApp.tsx index 452b8013d..34bebdf32 100644 --- a/ui/src/EntApp.tsx +++ b/ui/src/EntApp.tsx @@ -1,4 +1,4 @@ -import React, {useEffect, useState} from 'react'; +import React, {useCallback, useEffect, useState} from 'react'; import './App.sass'; import {TrafficPage} from "./components/TrafficPage"; import {TLSWarning} from "./components/TLSWarning/TLSWarning"; @@ -9,43 +9,29 @@ import InstallPage from "./components/InstallPage"; import LoginPage from "./components/LoginPage"; import LoadingOverlay from "./components/LoadingOverlay"; import AuthPageBase from './components/AuthPageBase'; +import entPageAtom, {Page} from "./recoil/entPage"; +import {useRecoilState} from "recoil"; const api = Api.getInstance(); -// TODO: move to state management -export enum Page { - Traffic, - Setup, - Login -} - -// TODO: move to state management -export interface MizuContextModel { - page: Page; - setPage: (page: Page) => void; -} - -// TODO: move to state management -export const MizuContext = React.createContext(null); - const EntApp = () => { const [isLoading, setIsLoading] = useState(true); const [showTLSWarning, setShowTLSWarning] = useState(false); const [userDismissedTLSWarning, setUserDismissedTLSWarning] = useState(false); const [addressesWithTLS, setAddressesWithTLS] = useState(new Set()); - const [page, setPage] = useState(Page.Traffic); // TODO: move to state management + const [entPage, setEntPage] = useRecoilState(entPageAtom); const [isFirstLogin, setIsFirstLogin] = useState(false); - const determinePage = async () => { // TODO: move to state management + const determinePage = useCallback(async () => { // TODO: move to state management try { const isInstallNeeded = await api.isInstallNeeded(); if (isInstallNeeded) { - setPage(Page.Setup); + setEntPage(Page.Setup); } else { const isAuthNeeded = await api.isAuthenticationNeeded(); if(isAuthNeeded) { - setPage(Page.Login); + setEntPage(Page.Login); } } } catch (e) { @@ -54,11 +40,11 @@ const EntApp = () => { } finally { setIsLoading(false); } - } + },[setEntPage]); useEffect(() => { determinePage(); - }, []); + }, [determinePage]); const onTLSDetected = (destAddress: string) => { addressesWithTLS.add(destAddress); @@ -71,7 +57,7 @@ const EntApp = () => { let pageComponent: any; - switch (page) { // TODO: move to state management / proper routing + switch (entPage) { // TODO: move to state management / proper routing case Page.Traffic: pageComponent = ; break; @@ -91,16 +77,14 @@ const EntApp = () => { return (
- - {page === Page.Traffic && } - {pageComponent} - {page === Page.Traffic && } - + {entPage === Page.Traffic && } + {pageComponent} + {entPage === Page.Traffic && }
); } diff --git a/ui/src/components/EntriesList.tsx b/ui/src/components/EntriesList.tsx index 4fb88cbf2..5aa7aac6e 100644 --- a/ui/src/components/EntriesList.tsx +++ b/ui/src/components/EntriesList.tsx @@ -6,11 +6,12 @@ import {EntryItem} from "./EntryListItem/EntryListItem"; import down from "./assets/downImg.svg"; import spinner from './assets/spinner.svg'; import Api from "../helpers/api"; +import {useRecoilState, useRecoilValue} from "recoil"; +import entriesAtom from "../recoil/entries"; +import wsConnectionAtom, {WsConnectionStatus} from "../recoil/wsConnection"; +import queryAtom from "../recoil/query"; interface EntriesListProps { - entries: any[]; - setEntries: any; - query: string; listEntryREF: any; onSnapBrokenEvent: () => void; isSnappedToBottom: boolean; @@ -22,12 +23,8 @@ interface EntriesListProps { startTime: number; noMoreDataTop: boolean; setNoMoreDataTop: (flag: boolean) => void; - focusedEntryId: string; - setFocusedEntryId: (id: string) => void; - updateQuery: any; leftOffTop: number; setLeftOffTop: (leftOffTop: number) => void; - isWebSocketConnectionClosed: boolean; ws: any; openWebSocket: (query: string, resetEntries: boolean) => void; leftOffBottom: number; @@ -38,7 +35,13 @@ interface EntriesListProps { const api = Api.getInstance(); -export const EntriesList: React.FC = ({entries, setEntries, query, listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, setQueriedCurrent, queriedTotal, setQueriedTotal, startTime, noMoreDataTop, setNoMoreDataTop, focusedEntryId, setFocusedEntryId, updateQuery, leftOffTop, setLeftOffTop, isWebSocketConnectionClosed, ws, openWebSocket, leftOffBottom, truncatedTimestamp, setTruncatedTimestamp, scrollableRef}) => { +export const EntriesList: React.FC = ({listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, setQueriedCurrent, queriedTotal, setQueriedTotal, startTime, noMoreDataTop, setNoMoreDataTop, leftOffTop, setLeftOffTop, ws, openWebSocket, leftOffBottom, truncatedTimestamp, setTruncatedTimestamp, scrollableRef}) => { + + const [entries, setEntries] = useRecoilState(entriesAtom); + const wsConnection = useRecoilValue(wsConnectionAtom); + const query = useRecoilValue(queryAtom); + const isWsConnectionClosed = wsConnection === WsConnectionStatus.Closed; + const [loadMoreTop, setLoadMoreTop] = useState(false); const [isLoadingTop, setIsLoadingTop] = useState(false); @@ -95,9 +98,9 @@ export const EntriesList: React.FC = ({entries, setEntries, qu },[setLoadMoreTop, setIsLoadingTop, entries, setEntries, query, setNoMoreDataTop, leftOffTop, setLeftOffTop, queriedCurrent, setQueriedCurrent, setQueriedTotal, setTruncatedTimestamp, scrollableRef]); useEffect(() => { - if(!isWebSocketConnectionClosed || !loadMoreTop || noMoreDataTop) return; + if(!isWsConnectionClosed || !loadMoreTop || noMoreDataTop) return; getOldEntries(); - }, [loadMoreTop, noMoreDataTop, getOldEntries, isWebSocketConnectionClosed]); + }, [loadMoreTop, noMoreDataTop, getOldEntries, isWsConnectionClosed]); const scrollbarVisible = scrollableRef.current?.childWrapperRef.current.clientHeight > scrollableRef.current?.wrapperRef.current.clientHeight; @@ -113,10 +116,7 @@ export const EntriesList: React.FC = ({entries, setEntries, qu {memoizedEntries.map(entry => )} @@ -131,9 +131,9 @@ export const EntriesList: React.FC = ({entries, setEntries, qu