diff --git a/.gitignore b/.gitignore index 4d5c55b7e..e3efed6b5 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ build # pprof pprof/* + +# Database Files +*.bin diff --git a/Dockerfile b/Dockerfile index 65b7b46ea..9345df30c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ FROM golang:1.16-alpine AS builder # Set necessary environment variables needed for our image. ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64 -RUN apk add libpcap-dev gcc g++ make bash +RUN apk add libpcap-dev gcc g++ make bash perl-utils # Move to agent working directory (/agent-build). WORKDIR /app/agent-build @@ -24,7 +24,7 @@ COPY tap/go.mod tap/go.mod ../tap/ COPY tap/api/go.* ../tap/api/ RUN go mod download # cheap trick to make the build faster (As long as go.mod wasn't changes) -RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get +RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get ARG COMMIT_HASH ARG GIT_BRANCH @@ -41,16 +41,24 @@ RUN go build -ldflags="-s -w \ -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \ -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent . +# Download Basenine executable, verify the sha1sum and move it to a directory in $PATH +ADD https://github.com/up9inc/basenine/releases/download/v0.2.6/basenine_linux_amd64 ./basenine_linux_amd64 +ADD https://github.com/up9inc/basenine/releases/download/v0.2.6/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256 +RUN shasum -a 256 -c basenine_linux_amd64.sha256 +RUN chmod +x ./basenine_linux_amd64 + COPY devops/build_extensions.sh .. RUN cd .. && /bin/bash build_extensions.sh FROM alpine:3.14 RUN apk add bash libpcap-dev tcpdump + WORKDIR /app # Copy binary and config files from /build to root folder of scratch container. COPY --from=builder ["/app/agent-build/mizuagent", "."] +COPY --from=builder ["/app/agent-build/basenine_linux_amd64", "/usr/local/bin/basenine"] COPY --from=builder ["/app/agent/build/extensions", "extensions"] COPY --from=site-build ["/app/ui-build/build", "site"] RUN mkdir /app/data/ diff --git a/acceptanceTests/tap_test.go b/acceptanceTests/tap_test.go index 02bca771f..ebee856de 100644 --- a/acceptanceTests/tap_test.go +++ b/acceptanceTests/tap_test.go @@ -472,7 +472,7 @@ func TestTapRedact(t *testing.T) { entryPayload := entryRequest["payload"].(map[string]interface{}) entryDetails := entryPayload["details"].(map[string]interface{}) - headers := entryDetails["headers"].([]interface{}) + headers := entryDetails["_headers"].([]interface{}) for _, headerInterface := range headers { header := headerInterface.(map[string]interface{}) if header["name"].(string) != "User-Agent" { @@ -587,7 +587,7 @@ func TestTapNoRedact(t *testing.T) { entryPayload := entryRequest["payload"].(map[string]interface{}) entryDetails := entryPayload["details"].(map[string]interface{}) - headers := entryDetails["headers"].([]interface{}) + headers := entryDetails["_headers"].([]interface{}) for _, headerInterface := range headers { header := headerInterface.(map[string]interface{}) if header["name"].(string) != "User-Agent" { @@ -808,7 +808,7 @@ func TestTapIgnoredUserAgents(t *testing.T) { entryPayload := entryRequest["payload"].(map[string]interface{}) entryDetails := entryPayload["details"].(map[string]interface{}) - entryHeaders := entryDetails["headers"].([]interface{}) + entryHeaders := entryDetails["_headers"].([]interface{}) for _, headerInterface := range entryHeaders { header := headerInterface.(map[string]interface{}) if header["name"].(string) != ignoredUserAgentCustomHeader { diff --git a/agent/go.mod b/agent/go.mod index 905c80f0b..f1af6b2fb 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -3,8 +3,8 @@ module mizuserver go 1.16 require ( + github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b github.com/djherbis/atime v1.0.0 - github.com/fsnotify/fsnotify v1.4.9 github.com/getkin/kin-openapi v0.76.0 github.com/gin-contrib/static v0.0.1 github.com/gin-gonic/gin v1.7.2 @@ -16,13 +16,12 @@ require ( github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 github.com/patrickmn/go-cache v2.1.0+incompatible + github.com/up9inc/basenine/client/go v0.0.0-20211106180626-0193408db715 github.com/up9inc/mizu/shared v0.0.0 github.com/up9inc/mizu/tap v0.0.0 github.com/up9inc/mizu/tap/api v0.0.0 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 - go.mongodb.org/mongo-driver v1.7.1 - gorm.io/driver/sqlite v1.1.4 - gorm.io/gorm v1.21.8 + golang.org/x/text v0.3.5 // indirect k8s.io/api v0.21.2 k8s.io/apimachinery v0.21.2 k8s.io/client-go v0.21.2 diff --git a/agent/go.sum b/agent/go.sum index 52d06e485..fe9b06095 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -52,6 +52,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b h1:8m+eVxVVDDyJFidv7Ck1OwqnDaQR6pTSRGlCC2Dnw0A= +github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b/go.mod h1:+tQQjzrp2501Nd6JXrb9s/XsNvFK3ZbxOnCdQl/vDRo= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -94,7 +96,6 @@ github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg= github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= @@ -111,7 +112,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/getkin/kin-openapi v0.76.0 h1:j77zg3Ec+k+r+GA3d8hBoXpAc6KX9TbBPrwQGBIy2sY= @@ -195,31 +195,7 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E= github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -251,7 +227,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -321,12 +296,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= -github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= -github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -336,14 +305,10 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -364,16 +329,12 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ= -github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -395,7 +356,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= @@ -419,7 +379,6 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -447,8 +406,6 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -458,8 +415,6 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -468,7 +423,6 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -485,8 +439,9 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -495,26 +450,22 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/up9inc/basenine/client/go v0.0.0-20211106180626-0193408db715 h1:3RNTMQZO/4g5gRn4R98cPwCjCrsMklmcOS0g+QwCh5c= +github.com/up9inc/basenine/client/go v0.0.0-20211106180626-0193408db715/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.7.1 h1:jwqTeEM3x6L9xDXrCxN0Hbg7vdGfPBOTIkr0+/LYZDA= -go.mongodb.org/mongo-driver v1.7.1/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -527,13 +478,11 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= @@ -611,7 +560,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -625,13 +573,10 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -685,13 +630,9 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -805,11 +746,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM= -gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw= -gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.21.8 h1:2CEwZSzogdhsKPlJ9OvBKTdlWIpELXb6HbfLfMNhSYI= -gorm.io/gorm v1.21.8/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/agent/main.go b/agent/main.go index 51ffc9a41..907920203 100644 --- a/agent/main.go +++ b/agent/main.go @@ -6,13 +6,10 @@ import ( "errors" "flag" "fmt" - "github.com/up9inc/mizu/shared/kubernetes" "io/ioutil" - v1 "k8s.io/api/core/v1" "mizuserver/pkg/api" "mizuserver/pkg/config" "mizuserver/pkg/controllers" - "mizuserver/pkg/database" "mizuserver/pkg/models" "mizuserver/pkg/providers" "mizuserver/pkg/routes" @@ -20,6 +17,7 @@ import ( "mizuserver/pkg/utils" "net/http" "os" + "os/exec" "os/signal" "path" "path/filepath" @@ -28,10 +26,15 @@ import ( "syscall" "time" + "github.com/up9inc/mizu/shared/kubernetes" + v1 "k8s.io/api/core/v1" + + "github.com/antelman107/net-wait-go/wait" "github.com/gin-contrib/static" "github.com/gin-gonic/gin" "github.com/gorilla/websocket" "github.com/op/go-logging" + basenine "github.com/up9inc/basenine/client/go" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" "github.com/up9inc/mizu/tap" @@ -49,10 +52,12 @@ var harsDir = flag.String("hars-dir", "", "Directory to read hars from") var extensions []*tapApi.Extension // global var extensionsMap map[string]*tapApi.Extension // global +var startTime int64 + const ( - socketConnectionRetries = 10 + socketConnectionRetries = 10 socketConnectionRetryDelay = time.Second * 2 - socketHandshakeTimeout = time.Second * 2 + socketHandshakeTimeout = time.Second * 2 ) func main() { @@ -109,7 +114,8 @@ func main() { go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel) } else if *apiServerMode { - database.InitDataBase(config.Config.AgentDatabasePath) + startBasenineServer(shared.BasenineHost, shared.BaseninePort) + startTime = time.Now().UnixNano() / int64(time.Millisecond) api.StartResolving(*namespace) outputItemsChannel := make(chan *tapApi.OutputChannelItem) @@ -142,6 +148,53 @@ func main() { logger.Log.Info("Exiting") } +func startBasenineServer(host string, port string) { + cmd := exec.Command("basenine", "-addr", host, "-port", port, "-persistent") + cmd.Dir = config.Config.AgentDatabasePath + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Start() + if err != nil { + logger.Log.Panicf("Failed starting Basenine: %v", err) + } + + if !wait.New( + wait.WithProto("tcp"), + wait.WithWait(200*time.Millisecond), + wait.WithBreak(50*time.Millisecond), + wait.WithDeadline(5*time.Second), + wait.WithDebug(true), + ).Do([]string{fmt.Sprintf("%s:%s", host, port)}) { + logger.Log.Panicf("Basenine is not available: %v", err) + } + + // Make a channel to gracefully exit Basenine. + channel := make(chan os.Signal) + signal.Notify(channel, os.Interrupt, syscall.SIGTERM) + + // Handle the channel. + go func() { + <-channel + cmd.Process.Signal(syscall.SIGTERM) + }() + + // Limit the database size to default 200MB + err = basenine.Limit(host, port, config.Config.MaxDBSizeBytes) + if err != nil { + logger.Log.Panicf("Error while limiting database size: %v", err) + } + + for _, extension := range extensions { + macros := extension.Dissector.Macros() + for macro, expanded := range macros { + err = basenine.Macro(host, port, macro, expanded) + if err != nil { + logger.Log.Panicf("Error while adding a macro: %v", err) + } + } + } +} + func loadExtensions() { dir, _ := filepath.Abs(filepath.Dir(os.Args[0])) extensionsDir := path.Join(dir, "./extensions/") @@ -200,7 +253,8 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) { app.Use(static.ServeRoot("/", "./site")) app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before - api.WebSocketRoutes(app, &eventHandlers) + api.WebSocketRoutes(app, &eventHandlers, startTime) + routes.QueryRoutes(app) routes.EntriesRoutes(app) routes.MetadataRoutes(app) routes.StatusRoutes(app) @@ -361,7 +415,7 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time. socketConnection, _, err := dialer.Dial(socketAddress, nil) if err != nil { if i < retryAmount { - logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay / time.Second) + logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay/time.Second) time.Sleep(retryDelay) } } else { @@ -371,8 +425,7 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time. return nil, lastErr } - -func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, error){ +func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, error) { provider, err := kubernetes.NewProviderInCluster() if err != nil { return nil, err diff --git a/agent/pkg/api/main.go b/agent/pkg/api/main.go index 56968ff3c..8bfc65787 100644 --- a/agent/pkg/api/main.go +++ b/agent/pkg/api/main.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "mizuserver/pkg/database" "mizuserver/pkg/holder" "mizuserver/pkg/providers" "os" @@ -14,15 +13,16 @@ import ( "strings" "time" - "go.mongodb.org/mongo-driver/bson/primitive" - "github.com/google/martian/har" + "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" tapApi "github.com/up9inc/mizu/tap/api" "mizuserver/pkg/models" "mizuserver/pkg/resolver" "mizuserver/pkg/utils" + + basenine "github.com/up9inc/basenine/client/go" ) var k8sResolver *resolver.Resolver @@ -99,6 +99,12 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension panic("Channel of captured messages is nil") } + connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort) + if err != nil { + panic(err) + } + connection.InsertMode() + disableOASValidation := false ctx := context.Background() doc, contractContent, router, err := loadOAS(ctx) @@ -112,13 +118,13 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension extension := extensionsMap[item.Protocol.Name] resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo) - mizuEntry := extension.Dissector.Analyze(item, primitive.NewObjectID().Hex(), resolvedSource, resolvedDestionation) + mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation) baseEntry := extension.Dissector.Summarize(mizuEntry) - mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry) + mizuEntry.Base = baseEntry if extension.Protocol.Name == "http" { if !disableOASValidation { var httpPair tapApi.HTTPRequestResponsePair - json.Unmarshal([]byte(mizuEntry.Entry), &httpPair) + json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair) contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent) baseEntry.ContractStatus = contract.Status @@ -128,18 +134,18 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension mizuEntry.ContractContent = contract.Content } - var pair tapApi.RequestResponsePair - json.Unmarshal([]byte(mizuEntry.Entry), &pair) - harEntry, err := utils.NewEntry(&pair) + harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime) if err == nil { rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Service) baseEntry.Rules = rules } } - database.CreateEntry(mizuEntry) - baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(baseEntry) - BroadcastToBrowserClients(baseEntryBytes) + data, err := json.Marshal(mizuEntry) + if err != nil { + panic(err) + } + connection.SendText(string(data)) } } @@ -171,21 +177,3 @@ func CheckIsServiceIP(address string) bool { } return k8sResolver.CheckIsServiceIP(address) } - -// gives a rough estimate of the size this will take up in the db, good enough for maintaining db size limit accurately -func getEstimatedEntrySizeBytes(mizuEntry *tapApi.MizuEntry) int { - sizeBytes := len(mizuEntry.Entry) - sizeBytes += len(mizuEntry.EntryId) - sizeBytes += len(mizuEntry.Service) - sizeBytes += len(mizuEntry.Url) - sizeBytes += len(mizuEntry.Method) - sizeBytes += len(mizuEntry.RequestSenderIp) - sizeBytes += len(mizuEntry.ResolvedDestination) - sizeBytes += len(mizuEntry.ResolvedSource) - sizeBytes += 8 // Status bytes (sqlite integer is always 8 bytes) - sizeBytes += 8 // Timestamp bytes - sizeBytes += 8 // SizeBytes bytes - sizeBytes += 1 // IsOutgoing bytes - - return sizeBytes -} diff --git a/agent/pkg/api/socket_routes.go b/agent/pkg/api/socket_routes.go index 7e765268c..dff86157f 100644 --- a/agent/pkg/api/socket_routes.go +++ b/agent/pkg/api/socket_routes.go @@ -1,13 +1,18 @@ package api import ( + "encoding/json" "errors" + "fmt" + "mizuserver/pkg/models" "net/http" "sync" "time" "github.com/gin-gonic/gin" "github.com/gorilla/websocket" + basenine "github.com/up9inc/basenine/client/go" + "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/debounce" "github.com/up9inc/mizu/shared/logger" ) @@ -39,17 +44,17 @@ func init() { connectedWebsockets = make(map[int]*SocketConnection, 0) } -func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) { +func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) { app.GET("/ws", func(c *gin.Context) { - websocketHandler(c.Writer, c.Request, eventHandlers, false) + websocketHandler(c.Writer, c.Request, eventHandlers, false, startTime) }) app.GET("/wsTapper", func(c *gin.Context) { - websocketHandler(c.Writer, c.Request, eventHandlers, true) + websocketHandler(c.Writer, c.Request, eventHandlers, true, startTime) }) } -func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool) { - conn, err := websocketUpgrader.Upgrade(w, r, nil) +func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool, startTime int64) { + ws, err := websocketUpgrader.Upgrade(w, r, nil) if err != nil { logger.Log.Errorf("Failed to set websocket upgrade: %v", err) return @@ -59,23 +64,103 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even connectedWebsocketIdCounter++ socketId := connectedWebsocketIdCounter - connectedWebsockets[socketId] = &SocketConnection{connection: conn, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper} + connectedWebsockets[socketId] = &SocketConnection{connection: ws, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper} websocketIdsLock.Unlock() + var connection *basenine.Connection + var isQuerySet bool + + // `!isTapper` means it's a connection from the web UI + if !isTapper { + connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort) + if err != nil { + panic(err) + } + } + + data := make(chan []byte) + meta := make(chan []byte) + defer func() { + data <- []byte(basenine.CloseChannel) + meta <- []byte(basenine.CloseChannel) + connection.Close() socketCleanup(socketId, connectedWebsockets[socketId]) }() eventHandlers.WebSocketConnect(socketId, isTapper) + startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(startTime) + BroadcastToBrowserClients(startTimeBytes) + for { - _, msg, err := conn.ReadMessage() + _, msg, err := ws.ReadMessage() if err != nil { logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err) break } - eventHandlers.WebSocketMessage(socketId, msg) + + if !isTapper && !isQuerySet { + query := string(msg) + err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query) + if err != nil { + toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{ + Type: "error", + AutoClose: 5000, + Text: fmt.Sprintf("Syntax error: %s", err.Error()), + }) + BroadcastToBrowserClients(toastBytes) + break + } + + isQuerySet = true + + handleDataChannel := func(c *basenine.Connection, data chan []byte) { + for { + bytes := <-data + + if string(bytes) == basenine.CloseChannel { + return + } + + var dataMap map[string]interface{} + err = json.Unmarshal(bytes, &dataMap) + + base := dataMap["base"].(map[string]interface{}) + base["id"] = uint(dataMap["id"].(float64)) + + baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base) + BroadcastToBrowserClients(baseEntryBytes) + } + } + + handleMetaChannel := func(c *basenine.Connection, meta chan []byte) { + for { + bytes := <-meta + + if string(bytes) == basenine.CloseChannel { + return + } + + var metadata *basenine.Metadata + err = json.Unmarshal(bytes, &metadata) + if err != nil { + logger.Log.Debugf("Error recieving metadata: %v\n", err.Error()) + } + + metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata) + BroadcastToBrowserClients(metadataBytes) + } + } + + go handleDataChannel(connection, data) + go handleMetaChannel(connection, meta) + + connection.Query(query, data, meta) + } else { + eventHandlers.WebSocketMessage(socketId, msg) + } } } diff --git a/agent/pkg/controllers/entries_controller.go b/agent/pkg/controllers/entries_controller.go index 32422d5e4..8b900b23d 100644 --- a/agent/pkg/controllers/entries_controller.go +++ b/agent/pkg/controllers/entries_controller.go @@ -2,14 +2,17 @@ package controllers import ( "encoding/json" - "fmt" - "github.com/gin-gonic/gin" - tapApi "github.com/up9inc/mizu/tap/api" - "mizuserver/pkg/database" "mizuserver/pkg/models" "mizuserver/pkg/utils" - "mizuserver/pkg/validation" "net/http" + "strconv" + + "github.com/gin-gonic/gin" + + basenine "github.com/up9inc/basenine/client/go" + "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/shared/logger" + tapApi "github.com/up9inc/mizu/tap/api" ) var extensionsMap map[string]*tapApi.Extension // global @@ -18,68 +21,36 @@ func InitExtensionsMap(ref map[string]*tapApi.Extension) { extensionsMap = ref } -func GetEntries(c *gin.Context) { - entriesFilter := &models.EntriesFilter{} - - if err := c.BindQuery(entriesFilter); err != nil { - c.JSON(http.StatusBadRequest, err) - } - err := validation.Validate(entriesFilter) +func Error(c *gin.Context, err error) bool { if err != nil { - c.JSON(http.StatusBadRequest, err) + logger.Log.Errorf("Error getting entry: %v", err) + c.Error(err) + c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": true, "msg": err.Error()}) + return true // signal that there was an error and the caller should return } - - order := database.OperatorToOrderMapping[entriesFilter.Operator] - operatorSymbol := database.OperatorToSymbolMapping[entriesFilter.Operator] - var entries []tapApi.MizuEntry - database.GetEntriesTable(). - Order(fmt.Sprintf("timestamp %s", order)). - Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)). - Limit(entriesFilter.Limit). - Find(&entries) - - if len(entries) > 0 && order == database.OrderDesc { - // the entries always order from oldest to newest - we should reverse - utils.ReverseSlice(entries) - } - - baseEntries := make([]tapApi.BaseEntryDetails, 0) - for _, entry := range entries { - baseEntryDetails := tapApi.BaseEntryDetails{} - if err := models.GetEntry(&entry, &baseEntryDetails); err != nil { - continue - } - - var pair tapApi.RequestResponsePair - json.Unmarshal([]byte(entry.Entry), &pair) - harEntry, err := utils.NewEntry(&pair) - if err == nil { - rules, _, _ := models.RunValidationRulesState(*harEntry, entry.Service) - baseEntryDetails.Rules = rules - } - - baseEntries = append(baseEntries, baseEntryDetails) - } - - c.JSON(http.StatusOK, baseEntries) + return false // no error, can continue } func GetEntry(c *gin.Context) { - var entryData tapApi.MizuEntry - database.GetEntriesTable(). - Where(map[string]string{"entryId": c.Param("entryId")}). - First(&entryData) + id, _ := strconv.Atoi(c.Param("id")) + var entry tapApi.MizuEntry + bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id) + if Error(c, err) { + return // exit + } + err = json.Unmarshal(bytes, &entry) + if Error(c, err) { + return // exit + } - extension := extensionsMap[entryData.ProtocolName] - protocol, representation, bodySize, _ := extension.Dissector.Represent(&entryData) + extension := extensionsMap[entry.Protocol.Name] + protocol, representation, bodySize, _ := extension.Dissector.Represent(entry.Protocol, entry.Request, entry.Response) var rules []map[string]interface{} var isRulesEnabled bool - if entryData.ProtocolName == "http" { - var pair tapApi.RequestResponsePair - json.Unmarshal([]byte(entryData.Entry), &pair) - harEntry, _ := utils.NewEntry(&pair) - _, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entryData.Service) + if entry.Protocol.Name == "http" { + harEntry, _ := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime) + _, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Service) isRulesEnabled = _isRulesEnabled inrec, _ := json.Marshal(rulesMatched) json.Unmarshal(inrec, &rules) @@ -89,7 +60,7 @@ func GetEntry(c *gin.Context) { Protocol: protocol, Representation: string(representation), BodySize: bodySize, - Data: entryData, + Data: entry, Rules: rules, IsRulesEnabled: isRulesEnabled, }) diff --git a/agent/pkg/controllers/query_controller.go b/agent/pkg/controllers/query_controller.go new file mode 100644 index 000000000..f74230f8b --- /dev/null +++ b/agent/pkg/controllers/query_controller.go @@ -0,0 +1,31 @@ +package controllers + +import ( + "net/http" + + "github.com/gin-gonic/gin" + basenine "github.com/up9inc/basenine/client/go" + "github.com/up9inc/mizu/shared" +) + +type ValidateResponse struct { + Valid bool `json:"valid"` + Message string `json:"message"` +} + +func PostValidate(c *gin.Context) { + query := c.PostForm("query") + valid := true + message := "" + + err := basenine.Validate(shared.BasenineHost, shared.BaseninePort, query) + if err != nil { + valid = false + message = err.Error() + } + + c.JSON(http.StatusOK, ValidateResponse{ + Valid: valid, + Message: message, + }) +} diff --git a/agent/pkg/database/main.go b/agent/pkg/database/main.go deleted file mode 100644 index 6d199823c..000000000 --- a/agent/pkg/database/main.go +++ /dev/null @@ -1,78 +0,0 @@ -package database - -import ( - "fmt" - "mizuserver/pkg/utils" - "time" - - "gorm.io/driver/sqlite" - "gorm.io/gorm" - "gorm.io/gorm/logger" - - tapApi "github.com/up9inc/mizu/tap/api" -) - -const ( - OrderDesc = "desc" - OrderAsc = "asc" - LT = "lt" - GT = "gt" - TimeFormat = "2006-01-02 15:04:05.000000000" -) - -var ( - DB *gorm.DB - IsDBLocked = false - OperatorToSymbolMapping = map[string]string{ - LT: "<", - GT: ">", - } - OperatorToOrderMapping = map[string]string{ - LT: OrderDesc, - GT: OrderAsc, - } -) - -var DBPath string - -func GetEntriesTable() *gorm.DB { - return DB.Table("mizu_entries") -} - -func CreateEntry(entry *tapApi.MizuEntry) { - if IsDBLocked { - return - } - GetEntriesTable().Create(entry) -} - -func InitDataBase(databasePath string) *gorm.DB { - DBPath = databasePath - DB, _ = gorm.Open(sqlite.Open(databasePath), &gorm.Config{ - Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond}, - }) - _ = DB.AutoMigrate(&tapApi.MizuEntry{}) // this will ensure table is created - go StartEnforcingDatabaseSize() - return DB -} - -func GetEntriesFromDb(timeFrom time.Time, timeTo time.Time, protocolName *string) []tapApi.MizuEntry { - order := OrderDesc - protocolNameCondition := "1 = 1" - if protocolName != nil { - protocolNameCondition = fmt.Sprintf("protocolName = '%s'", *protocolName) - } - - var entries []tapApi.MizuEntry - GetEntriesTable(). - Where(protocolNameCondition). - Where(fmt.Sprintf("created_at BETWEEN '%s' AND '%s'", timeFrom.Format(TimeFormat), timeTo.Format(TimeFormat))). - Order(fmt.Sprintf("timestamp %s", order)). - Find(&entries) - - if len(entries) > 0 { - // the entries always order from oldest to newest so we should revers - utils.ReverseSlice(entries) - } - return entries -} diff --git a/agent/pkg/database/size_enforcer.go b/agent/pkg/database/size_enforcer.go deleted file mode 100644 index 7e06da7fe..000000000 --- a/agent/pkg/database/size_enforcer.go +++ /dev/null @@ -1,102 +0,0 @@ -package database - -import ( - "mizuserver/pkg/config" - "os" - "time" - - "github.com/fsnotify/fsnotify" - "github.com/up9inc/mizu/shared/debounce" - "github.com/up9inc/mizu/shared/logger" - "github.com/up9inc/mizu/shared/units" - tapApi "github.com/up9inc/mizu/tap/api" -) - -const percentageOfMaxSizeBytesToPrune = 15 - -func StartEnforcingDatabaseSize() { - watcher, err := fsnotify.NewWatcher() - if err != nil { - logger.Log.Fatalf("Error creating filesystem watcher for db size enforcement: %v\n", err) - return - } - - checkFileSizeDebouncer := debounce.NewDebouncer(5*time.Second, func() { - checkFileSize(config.Config.MaxDBSizeBytes) - }) - - go func() { - for { - select { - case event, ok := <-watcher.Events: - if !ok { - return // closed channel - } - if event.Op == fsnotify.Write { - checkFileSizeDebouncer.SetOn() - } - case err, ok := <-watcher.Errors: - if !ok { - return // closed channel - } - logger.Log.Errorf("filesystem watcher encountered error:%v", err) - } - } - }() - - err = watcher.Add(DBPath) - if err != nil { - logger.Log.Fatalf("Error adding %s to filesystem watcher for db size enforcement: %v\n", DBPath, err) - } -} - -func checkFileSize(maxSizeBytes int64) { - fileStat, err := os.Stat(DBPath) - if err != nil { - logger.Log.Errorf("Error checking %s file size: %v", DBPath, err) - } else { - if fileStat.Size() > maxSizeBytes { - pruneOldEntries(fileStat.Size()) - } - } -} - -func pruneOldEntries(currentFileSize int64) { - // sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here - IsDBLocked = true - defer func() { IsDBLocked = false }() - - amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune) - - rows, err := GetEntriesTable().Limit(10000).Order("id").Rows() - if err != nil { - logger.Log.Errorf("Error getting 10000 first db rows: %v", err) - return - } - - entryIdsToRemove := make([]uint, 0) - bytesToBeRemoved := int64(0) - for rows.Next() { - if bytesToBeRemoved >= amountOfBytesToTrim { - break - } - var entry tapApi.MizuEntry - err = DB.ScanRows(rows, &entry) - if err != nil { - logger.Log.Errorf("Error scanning db row: %v", err) - continue - } - - entryIdsToRemove = append(entryIdsToRemove, entry.ID) - bytesToBeRemoved += int64(entry.EstimatedSizeBytes) - } - - if len(entryIdsToRemove) > 0 { - GetEntriesTable().Where(entryIdsToRemove).Delete(tapApi.MizuEntry{}) - // VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this - DB.Exec("VACUUM") - logger.Log.Errorf("Removed %d rows and cleared %s", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved)) - } else { - logger.Log.Error("Found no rows to remove when pruning") - } -} diff --git a/agent/pkg/models/models.go b/agent/pkg/models/models.go index 0209f3675..6add39eeb 100644 --- a/agent/pkg/models/models.go +++ b/agent/pkg/models/models.go @@ -2,12 +2,12 @@ package models import ( "encoding/json" + "mizuserver/pkg/rules" tapApi "github.com/up9inc/mizu/tap/api" - "mizuserver/pkg/rules" - "github.com/google/martian/har" + basenine "github.com/up9inc/basenine/client/go" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/tap" ) @@ -16,15 +16,9 @@ func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error { return v.UnmarshalData(r) } -type EntriesFilter struct { - Limit int `form:"limit" validate:"required,min=1,max=200"` - Operator string `form:"operator" validate:"required,oneof='lt' 'gt'"` - Timestamp int64 `form:"timestamp" validate:"required,min=1"` -} - type WebSocketEntryMessage struct { *shared.WebSocketMessageMetadata - Data *tapApi.BaseEntryDetails `json:"data,omitempty"` + Data map[string]interface{} `json:"data,omitempty"` } type WebSocketTappedEntryMessage struct { @@ -42,7 +36,28 @@ type AuthStatus struct { Model string `json:"model"` } -func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntryDetails) ([]byte, error) { +type ToastMessage struct { + Type string `json:"type"` + AutoClose uint `json:"autoClose"` + Text string `json:"text"` +} + +type WebSocketToastMessage struct { + *shared.WebSocketMessageMetadata + Data *ToastMessage `json:"data,omitempty"` +} + +type WebSocketQueryMetadataMessage struct { + *shared.WebSocketMessageMetadata + Data *basenine.Metadata `json:"data,omitempty"` +} + +type WebSocketStartTimeMessage struct { + *shared.WebSocketMessageMetadata + Data int64 `json:"data"` +} + +func CreateBaseEntryWebSocketMessage(base map[string]interface{}) ([]byte, error) { message := &WebSocketEntryMessage{ WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{ MessageType: shared.WebSocketMessageTypeEntry, @@ -72,6 +87,36 @@ func CreateWebsocketOutboundLinkMessage(base *tap.OutboundLink) ([]byte, error) return json.Marshal(message) } +func CreateWebsocketToastMessage(base *ToastMessage) ([]byte, error) { + message := &WebSocketToastMessage{ + WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{ + MessageType: shared.WebSocketMessageTypeToast, + }, + Data: base, + } + return json.Marshal(message) +} + +func CreateWebsocketQueryMetadataMessage(base *basenine.Metadata) ([]byte, error) { + message := &WebSocketQueryMetadataMessage{ + WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{ + MessageType: shared.WebSocketMessageTypeQueryMetadata, + }, + Data: base, + } + return json.Marshal(message) +} + +func CreateWebsocketStartTimeMessage(base int64) ([]byte, error) { + message := &WebSocketStartTimeMessage{ + WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{ + MessageType: shared.WebSocketMessageTypeStartTime, + }, + Data: base, + } + return json.Marshal(message) +} + // ExtendedHAR is the top level object of a HAR log. type ExtendedHAR struct { Log *ExtendedLog `json:"log"` diff --git a/agent/pkg/providers/stats_provider_test.go b/agent/pkg/providers/stats_provider_test.go index 4e0d3ff62..13acfece9 100644 --- a/agent/pkg/providers/stats_provider_test.go +++ b/agent/pkg/providers/stats_provider_test.go @@ -2,16 +2,10 @@ package providers_test import ( "fmt" - "mizuserver/pkg/config" - "mizuserver/pkg/database" "mizuserver/pkg/providers" "testing" ) -func init() { - database.InitDataBase(config.DefaultDatabasePath) -} - func TestNoEntryAddedCount(t *testing.T) { entriesStats := providers.GetGeneralStats() diff --git a/agent/pkg/routes/entries_routes.go b/agent/pkg/routes/entries_routes.go index 8e578fea4..12a597461 100644 --- a/agent/pkg/routes/entries_routes.go +++ b/agent/pkg/routes/entries_routes.go @@ -1,14 +1,14 @@ package routes import ( - "github.com/gin-gonic/gin" "mizuserver/pkg/controllers" + + "github.com/gin-gonic/gin" ) // EntriesRoutes defines the group of har entries routes. func EntriesRoutes(ginApp *gin.Engine) { routeGroup := ginApp.Group("/entries") - routeGroup.GET("/", controllers.GetEntries) // get entries (base/thin entries) - routeGroup.GET("/:entryId", controllers.GetEntry) // get single (full) entry + routeGroup.GET("/:id", controllers.GetEntry) // get single (full) entry } diff --git a/agent/pkg/routes/query_routes.go b/agent/pkg/routes/query_routes.go new file mode 100644 index 000000000..2807bd0b6 --- /dev/null +++ b/agent/pkg/routes/query_routes.go @@ -0,0 +1,13 @@ +package routes + +import ( + "mizuserver/pkg/controllers" + + "github.com/gin-gonic/gin" +) + +func QueryRoutes(ginApp *gin.Engine) { + routeGroup := ginApp.Group("/query") + + routeGroup.POST("/validate", controllers.PostValidate) +} diff --git a/agent/pkg/sensitiveDataFiltering/consts.go b/agent/pkg/sensitiveDataFiltering/consts.go deleted file mode 100644 index e5624de73..000000000 --- a/agent/pkg/sensitiveDataFiltering/consts.go +++ /dev/null @@ -1,10 +0,0 @@ -package sensitiveDataFiltering - -const maskedFieldPlaceholderValue = "[REDACTED]" - -//these values MUST be all lower case and contain no `-` or `_` characters -var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password", - "username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt", - "bearer", "clientid", "clientsecret", "redirecturi", "phonenumber", - "zip", "zipcode", "address", "country", "firstname", "lastname", - "middlename", "fname", "lname", "birthdate"} diff --git a/agent/pkg/up9/main.go b/agent/pkg/up9/main.go index 6fa9304bd..49739bbac 100644 --- a/agent/pkg/up9/main.go +++ b/agent/pkg/up9/main.go @@ -7,15 +7,16 @@ import ( "encoding/json" "fmt" "io/ioutil" - "mizuserver/pkg/database" "mizuserver/pkg/utils" "net/http" "net/url" "regexp" "strings" + "sync" "time" "github.com/google/martian/har" + basenine "github.com/up9inc/basenine/client/go" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" tapApi "github.com/up9inc/mizu/tap/api" @@ -23,6 +24,7 @@ import ( const ( AnalyzeCheckSleepTime = 5 * time.Second + SentCountLogInterval = 100 ) type GuestToken struct { @@ -204,44 +206,62 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva analyzeInformation.AnalyzeDestination = envPrefix analyzeInformation.SentCount = 0 - sleepTime := time.Second * time.Duration(uploadIntervalSec) + // "http or grpc" filter indicates that we're only interested in HTTP and gRPC entries + query := "http or grpc" - var timeFrom time.Time - protocolFilter := "http" + logger.Log.Infof("Getting entries from the database\n") - for { - timeTo := time.Now() - logger.Log.Infof("Getting entries from %v, to %v\n", timeFrom.Format(time.RFC3339Nano), timeTo.Format(time.RFC3339Nano)) - entriesArray := database.GetEntriesFromDb(timeFrom, timeTo, &protocolFilter) + var connection *basenine.Connection + var err error + connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort) + if err != nil { + panic(err) + } - if len(entriesArray) > 0 { - result := make([]har.Entry, 0) - for _, data := range entriesArray { - var pair tapApi.RequestResponsePair - if err := json.Unmarshal([]byte(data.Entry), &pair); err != nil { - continue - } - harEntry, err := utils.NewEntry(&pair) - if err != nil { - continue - } - if data.ResolvedSource != "" { - harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: data.ResolvedSource}) - } - if data.ResolvedDestination != "" { - harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: data.ResolvedDestination}) - harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, data.ResolvedDestination) - } + data := make(chan []byte) + meta := make(chan []byte) - // go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64 - if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil { - continue - } + defer func() { + data <- []byte(basenine.CloseChannel) + meta <- []byte(basenine.CloseChannel) + connection.Close() + }() - result = append(result, *harEntry) + handleDataChannel := func(wg *sync.WaitGroup, connection *basenine.Connection, data chan []byte) { + defer wg.Done() + for { + dataBytes := <-data + + if string(dataBytes) == basenine.CloseChannel { + return } - logger.Log.Infof("About to upload %v entries\n", len(result)) + var dataMap map[string]interface{} + err = json.Unmarshal(dataBytes, &dataMap) + + result := make([]har.Entry, 0) + var entry tapApi.MizuEntry + if err := json.Unmarshal([]byte(dataBytes), &entry); err != nil { + continue + } + harEntry, err := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime) + if err != nil { + continue + } + if entry.ResolvedSource != "" { + harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: entry.ResolvedSource}) + } + if entry.ResolvedDestination != "" { + harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entry.ResolvedDestination}) + harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entry.ResolvedDestination) + } + + // go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64 + if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil { + continue + } + + result = append(result, *harEntry) body, jMarshalErr := json.Marshal(result) if jMarshalErr != nil { @@ -273,18 +293,33 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva logger.Log.Info("Stopping sync entries") logger.Log.Fatal(postErr) } - analyzeInformation.SentCount += len(entriesArray) - logger.Log.Infof("Finish uploading %v entries to %s\n", len(entriesArray), GetTrafficDumpUrl(envPrefix, model)) + analyzeInformation.SentCount += 1 - logger.Log.Infof("Uploaded %v entries until now", analyzeInformation.SentCount) - } else { - logger.Log.Infof("Nothing to upload") + if analyzeInformation.SentCount%SentCountLogInterval == 0 { + logger.Log.Infof("Uploaded %v entries until now", analyzeInformation.SentCount) + } } - - logger.Log.Infof("Sleeping for %v...\n", sleepTime) - time.Sleep(sleepTime) - timeFrom = timeTo } + + handleMetaChannel := func(wg *sync.WaitGroup, connection *basenine.Connection, meta chan []byte) { + defer wg.Done() + for { + metaBytes := <-meta + + if string(metaBytes) == basenine.CloseChannel { + return + } + } + } + + var wg sync.WaitGroup + go handleDataChannel(&wg, connection, data) + go handleMetaChannel(&wg, connection, meta) + wg.Add(2) + + connection.Query(query, data, meta) + + wg.Wait() } func UpdateAnalyzeStatus(callback func(data []byte)) { diff --git a/agent/pkg/utils/har.go b/agent/pkg/utils/har.go index a1a06be78..108e91b7b 100644 --- a/agent/pkg/utils/har.go +++ b/agent/pkg/utils/har.go @@ -10,7 +10,6 @@ import ( "github.com/google/martian/har" "github.com/up9inc/mizu/shared/logger" - "github.com/up9inc/mizu/tap/api" ) // Keep it because we might want cookies in the future @@ -120,13 +119,11 @@ func BuildPostParams(rawParams []interface{}) []har.Param { return params } -func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error) { - reqDetails := request.Payload.(map[string]interface{})["details"].(map[string]interface{}) +func NewRequest(request map[string]interface{}) (harRequest *har.Request, err error) { + headers, host, scheme, authority, path, _ := BuildHeaders(request["_headers"].([]interface{})) + cookies := make([]har.Cookie, 0) // BuildCookies(request["_cookies"].([]interface{})) - headers, host, scheme, authority, path, _ := BuildHeaders(reqDetails["headers"].([]interface{})) - cookies := make([]har.Cookie, 0) // BuildCookies(reqDetails["cookies"].([]interface{})) - - postData, _ := reqDetails["postData"].(map[string]interface{}) + postData, _ := request["postData"].(map[string]interface{}) mimeType, _ := postData["mimeType"] if mimeType == nil || len(mimeType.(string)) == 0 { mimeType = "text/html" @@ -138,7 +135,7 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error } queryString := make([]har.QueryString, 0) - for _, _qs := range reqDetails["queryString"].([]interface{}) { + for _, _qs := range request["_queryString"].([]interface{}) { qs := _qs.(map[string]interface{}) queryString = append(queryString, har.QueryString{ Name: qs["name"].(string), @@ -146,7 +143,7 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error }) } - url := fmt.Sprintf("http://%s%s", host, reqDetails["url"].(string)) + url := fmt.Sprintf("http://%s%s", host, request["url"].(string)) if strings.HasPrefix(mimeType.(string), "application/grpc") { url = fmt.Sprintf("%s://%s%s", scheme, authority, path) } @@ -157,9 +154,9 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error } harRequest = &har.Request{ - Method: reqDetails["method"].(string), + Method: request["method"].(string), URL: url, - HTTPVersion: reqDetails["httpVersion"].(string), + HTTPVersion: request["httpVersion"].(string), HeadersSize: -1, BodySize: int64(bytes.NewBufferString(postDataText).Len()), QueryString: queryString, @@ -175,13 +172,11 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error return } -func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err error) { - resDetails := response.Payload.(map[string]interface{})["details"].(map[string]interface{}) +func NewResponse(response map[string]interface{}) (harResponse *har.Response, err error) { + headers, _, _, _, _, _status := BuildHeaders(response["_headers"].([]interface{})) + cookies := make([]har.Cookie, 0) // BuildCookies(response["_cookies"].([]interface{})) - headers, _, _, _, _, _status := BuildHeaders(resDetails["headers"].([]interface{})) - cookies := make([]har.Cookie, 0) // BuildCookies(resDetails["cookies"].([]interface{})) - - content, _ := resDetails["content"].(map[string]interface{}) + content, _ := response["content"].(map[string]interface{}) mimeType, _ := content["mimeType"] if mimeType == nil || len(mimeType.(string)) == 0 { mimeType = "text/html" @@ -200,9 +195,11 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e Size: int64(len(bodyText)), } - status := int(resDetails["status"].(float64)) + status := int(response["status"].(float64)) if strings.HasPrefix(mimeType.(string), "application/grpc") { - status, err = strconv.Atoi(_status) + if _status != "" { + status, err = strconv.Atoi(_status) + } if err != nil { logger.Log.Errorf("Failed converting status to int %s (%v,%+v)", err, err, err) return nil, errors.New("failed converting response status to int for HAR") @@ -210,9 +207,9 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e } harResponse = &har.Response{ - HTTPVersion: resDetails["httpVersion"].(string), + HTTPVersion: response["httpVersion"].(string), Status: status, - StatusText: resDetails["statusText"].(string), + StatusText: response["statusText"].(string), HeadersSize: -1, BodySize: int64(bytes.NewBufferString(bodyText).Len()), Headers: headers, @@ -222,34 +219,33 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e return } -func NewEntry(pair *api.RequestResponsePair) (*har.Entry, error) { - harRequest, err := NewRequest(&pair.Request) +func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*har.Entry, error) { + harRequest, err := NewRequest(request) if err != nil { logger.Log.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err) return nil, errors.New("failed converting request to HAR") } - harResponse, err := NewResponse(&pair.Response) + harResponse, err := NewResponse(response) if err != nil { logger.Log.Errorf("Failed converting response to HAR %s (%v,%+v)", err, err, err) return nil, errors.New("failed converting response to HAR") } - totalTime := pair.Response.CaptureTime.Sub(pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds() - if totalTime < 1 { - totalTime = 1 + if elapsedTime < 1 { + elapsedTime = 1 } harEntry := har.Entry{ - StartedDateTime: pair.Request.CaptureTime, - Time: totalTime, + StartedDateTime: startTime, + Time: elapsedTime, Request: harRequest, Response: harResponse, Cache: &har.Cache{}, Timings: &har.Timings{ Send: -1, Wait: -1, - Receive: totalTime, + Receive: elapsedTime, }, } diff --git a/agent/pkg/utils/truncating_logger.go b/agent/pkg/utils/truncating_logger.go deleted file mode 100644 index 8302d5512..000000000 --- a/agent/pkg/utils/truncating_logger.go +++ /dev/null @@ -1,60 +0,0 @@ -package utils - -import ( - "context" - "fmt" - "time" - - loggerShared "github.com/up9inc/mizu/shared/logger" - "gorm.io/gorm/logger" - "gorm.io/gorm/utils" -) - -// TruncatingLogger implements the gorm logger.Interface interface. Its purpose is to act as gorm's logger while truncating logs to a max of 50 characters to minimise the performance impact -type TruncatingLogger struct { - LogLevel logger.LogLevel - SlowThreshold time.Duration -} - -func (truncatingLogger *TruncatingLogger) LogMode(logLevel logger.LogLevel) logger.Interface { - truncatingLogger.LogLevel = logLevel - return truncatingLogger -} - -func (truncatingLogger *TruncatingLogger) Info(_ context.Context, message string, __ ...interface{}) { - if truncatingLogger.LogLevel < logger.Info { - return - } - loggerShared.Log.Errorf("gorm info: %.150s", message) -} - -func (truncatingLogger *TruncatingLogger) Warn(_ context.Context, message string, __ ...interface{}) { - if truncatingLogger.LogLevel < logger.Warn { - return - } - loggerShared.Log.Errorf("gorm warning: %.150s", message) -} - -func (truncatingLogger *TruncatingLogger) Error(_ context.Context, message string, __ ...interface{}) { - if truncatingLogger.LogLevel < logger.Error { - return - } - loggerShared.Log.Errorf("gorm error: %.150s", message) -} - -func (truncatingLogger *TruncatingLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) { - if truncatingLogger.LogLevel == logger.Silent { - return - } - elapsed := time.Since(begin) - if err != nil { - sql, rows := fc() // copied into every condition as this is a potentially heavy operation best done only when necessary - truncatingLogger.Error(ctx, fmt.Sprintf("Error in %s: %v - elapsed: %fs affected rows: %d, sql: %s", utils.FileWithLineNum(), err, elapsed.Seconds(), rows, sql)) - } else if truncatingLogger.LogLevel >= logger.Warn && elapsed > truncatingLogger.SlowThreshold { - sql, rows := fc() - truncatingLogger.Warn(ctx, fmt.Sprintf("Slow sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql)) - } else if truncatingLogger.LogLevel >= logger.Info { - sql, rows := fc() - truncatingLogger.Info(ctx, fmt.Sprintf("Sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql)) - } -} diff --git a/cli/config/config.go b/cli/config/config.go index 386f4816a..361d50068 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -3,14 +3,15 @@ package config import ( "errors" "fmt" - "github.com/up9inc/mizu/tap/api" "io/ioutil" - "k8s.io/apimachinery/pkg/util/json" "os" "reflect" "strconv" "strings" + "github.com/up9inc/mizu/tap/api" + "k8s.io/apimachinery/pkg/util/json" + "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" @@ -396,7 +397,7 @@ func getMizuAgentConfig(targetNamespaces []string, mizuApiFilteringOptions *api. TapperResources: Config.Tap.TapperResources, MizuResourcesNamespace: Config.MizuResourcesNamespace, MizuApiFilteringOptions: *mizuApiFilteringOptions, - AgentDatabasePath: fmt.Sprintf("%s%s", shared.DataDirPath, "entries.db"), + AgentDatabasePath: shared.DataDirPath, } return &config, nil } diff --git a/debug.Dockerfile b/debug.Dockerfile index 2c91746d5..d04e6b995 100644 --- a/debug.Dockerfile +++ b/debug.Dockerfile @@ -12,7 +12,7 @@ FROM golang:1.16-alpine AS builder # Set necessary environment variables needed for our image. ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64 -RUN apk add libpcap-dev gcc g++ make bash +RUN apk add libpcap-dev gcc g++ make bash perl-utils # Move to agent working directory (/agent-build). WORKDIR /app/agent-build @@ -23,7 +23,7 @@ COPY tap/go.mod tap/go.mod ../tap/ COPY tap/api/go.* ../tap/api/ RUN go mod download # cheap trick to make the build faster (As long as go.mod wasn't changes) -RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get +RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get ARG COMMIT_HASH ARG GIT_BRANCH @@ -36,6 +36,12 @@ COPY tap ../tap COPY agent . RUN go build -gcflags="all=-N -l" -o mizuagent . +# Download Basenine executable, verify the sha1sum and move it to a directory in $PATH +ADD https://github.com/up9inc/basenine/releases/download/v0.2.6/basenine_linux_amd64 ./basenine_linux_amd64 +ADD https://github.com/up9inc/basenine/releases/download/v0.2.6/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256 +RUN shasum -a 256 -c basenine_linux_amd64.sha256 +RUN chmod +x ./basenine_linux_amd64 + COPY devops/build_extensions_debug.sh .. RUN cd .. && /bin/bash build_extensions_debug.sh @@ -43,10 +49,12 @@ RUN cd .. && /bin/bash build_extensions_debug.sh FROM golang:1.16-alpine RUN apk add bash libpcap-dev tcpdump + WORKDIR /app # Copy binary and config files from /build to root folder of scratch container. COPY --from=builder ["/app/agent-build/mizuagent", "."] +COPY --from=builder ["/app/agent-build/basenine_linux_amd64", "/usr/local/bin/basenine"] COPY --from=builder ["/app/agent/build/extensions", "extensions"] COPY --from=site-build ["/app/ui-build/build", "site"] diff --git a/shared/consts.go b/shared/consts.go index 79e4b84e5..6dc3067b5 100644 --- a/shared/consts.go +++ b/shared/consts.go @@ -14,4 +14,6 @@ const ( GoGCEnvVar = "GOGC" DefaultApiServerPort = 8899 DebugModeEnvVar = "MIZU_DEBUG" + BasenineHost = "localhost" + BaseninePort = "9099" ) diff --git a/shared/models.go b/shared/models.go index 10b47883b..82b0abaef 100644 --- a/shared/models.go +++ b/shared/models.go @@ -17,6 +17,9 @@ const ( WebSocketMessageTypeUpdateStatus WebSocketMessageType = "status" WebSocketMessageTypeAnalyzeStatus WebSocketMessageType = "analyzeStatus" WebsocketMessageTypeOutboundLink WebSocketMessageType = "outboundLink" + WebSocketMessageTypeToast WebSocketMessageType = "toast" + WebSocketMessageTypeQueryMetadata WebSocketMessageType = "queryMetadata" + WebSocketMessageTypeStartTime WebSocketMessageType = "startTime" ) type Resources struct { diff --git a/tap/api/api.go b/tap/api/api.go index 5de34befe..3d1d552ef 100644 --- a/tap/api/api.go +++ b/tap/api/api.go @@ -18,7 +18,8 @@ import ( type Protocol struct { Name string `json:"name"` LongName string `json:"longName"` - Abbreviation string `json:"abbreviation"` + Abbreviation string `json:"abbr"` + Macro string `json:"macro"` Version string `json:"version"` BackgroundColor string `json:"backgroundColor"` ForegroundColor string `json:"foregroundColor"` @@ -28,6 +29,12 @@ type Protocol struct { Priority uint8 `json:"priority"` } +type TCP struct { + IP string `json:"ip"` + Port string `json:"port"` + Name string `json:"name"` +} + type Extension struct { Protocol *Protocol Path string @@ -74,6 +81,7 @@ type OutputChannelItem struct { Timestamp int64 ConnectionInfo *ConnectionInfo Pair *RequestResponsePair + Summary *BaseEntryDetails } type SuperTimer struct { @@ -89,9 +97,10 @@ type Dissector interface { Register(*Extension) Ping() Dissect(b *bufio.Reader, isClient bool, tcpID *TcpID, counterPair *CounterPair, superTimer *SuperTimer, superIdentifier *SuperIdentifier, emitter Emitter, options *TrafficFilteringOptions) error - Analyze(item *OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *MizuEntry + Analyze(item *OutputChannelItem, resolvedSource string, resolvedDestination string) *MizuEntry Summarize(entry *MizuEntry) *BaseEntryDetails - Represent(entry *MizuEntry) (protocol Protocol, object []byte, bodySize int64, err error) + Represent(pIn Protocol, request map[string]interface{}, response map[string]interface{}) (pOut Protocol, object []byte, bodySize int64, err error) + Macros() map[string]string } type Emitting struct { @@ -109,39 +118,36 @@ func (e *Emitting) Emit(item *OutputChannelItem) { } type MizuEntry struct { - ID uint `gorm:"primarykey"` - CreatedAt time.Time - UpdatedAt time.Time - ProtocolName string `json:"protocolName" gorm:"column:protocolName"` - ProtocolLongName string `json:"protocolLongName" gorm:"column:protocolLongName"` - ProtocolAbbreviation string `json:"protocolAbbreviation" gorm:"column:protocolAbbreviation"` - ProtocolVersion string `json:"protocolVersion" gorm:"column:protocolVersion"` - ProtocolBackgroundColor string `json:"protocolBackgroundColor" gorm:"column:protocolBackgroundColor"` - ProtocolForegroundColor string `json:"protocolForegroundColor" gorm:"column:protocolForegroundColor"` - ProtocolFontSize int8 `json:"protocolFontSize" gorm:"column:protocolFontSize"` - ProtocolReferenceLink string `json:"protocolReferenceLink" gorm:"column:protocolReferenceLink"` - Entry string `json:"entry,omitempty" gorm:"column:entry"` - EntryId string `json:"entryId" gorm:"column:entryId"` - Url string `json:"url" gorm:"column:url"` - Method string `json:"method" gorm:"column:method"` - Status int `json:"status" gorm:"column:status"` - RequestSenderIp string `json:"requestSenderIp" gorm:"column:requestSenderIp"` - Service string `json:"service" gorm:"column:service"` - Timestamp int64 `json:"timestamp" gorm:"column:timestamp"` - ElapsedTime int64 `json:"elapsedTime" gorm:"column:elapsedTime"` - Path string `json:"path" gorm:"column:path"` - ResolvedSource string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"` - ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` - SourceIp string `json:"sourceIp,omitempty" gorm:"column:sourceIp"` - DestinationIp string `json:"destinationIp,omitempty" gorm:"column:destinationIp"` - SourcePort string `json:"sourcePort,omitempty" gorm:"column:sourcePort"` - DestinationPort string `json:"destinationPort,omitempty" gorm:"column:destinationPort"` - IsOutgoing bool `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"` - ContractStatus ContractStatus `json:"contractStatus,omitempty" gorm:"column:contractStatus"` - ContractRequestReason string `json:"contractRequestReason,omitempty" gorm:"column:contractRequestReason"` - ContractResponseReason string `json:"contractResponseReason,omitempty" gorm:"column:contractResponseReason"` - ContractContent string `json:"contractContent,omitempty" gorm:"column:contractContent"` - EstimatedSizeBytes int `json:"-" gorm:"column:estimatedSizeBytes"` + Id uint `json:"id"` + Protocol Protocol `json:"proto"` + Source *TCP `json:"src"` + Destination *TCP `json:"dst"` + Outgoing bool `json:"outgoing"` + Timestamp int64 `json:"timestamp"` + StartTime time.Time `json:"startTime"` + Request map[string]interface{} `json:"request"` + Response map[string]interface{} `json:"response"` + Base *BaseEntryDetails `json:"base"` + Summary string `json:"summary"` + Url string `json:"url"` + Method string `json:"method"` + Status int `json:"status"` + RequestSenderIp string `json:"requestSenderIp"` + Service string `json:"service"` + ElapsedTime int64 `json:"elapsedTime"` + Path string `json:"path"` + ResolvedSource string `json:"resolvedSource,omitempty"` + ResolvedDestination string `json:"resolvedDestination,omitempty"` + SourceIp string `json:"sourceIp,omitempty"` + DestinationIp string `json:"destinationIp,omitempty"` + SourcePort string `json:"sourcePort,omitempty"` + DestinationPort string `json:"destinationPort,omitempty"` + IsOutgoing bool `json:"isOutgoing,omitempty"` + ContractStatus ContractStatus `json:"contractStatus,omitempty"` + ContractRequestReason string `json:"contractRequestReason,omitempty"` + ContractResponseReason string `json:"contractResponseReason,omitempty"` + ContractContent string `json:"contractContent,omitempty"` + HTTPPair string `json:"httpPair,omitempty"` } type MizuEntryWrapper struct { @@ -154,7 +160,7 @@ type MizuEntryWrapper struct { } type BaseEntryDetails struct { - Id string `json:"id,omitempty"` + Id uint `json:"id"` Protocol Protocol `json:"protocol,omitempty"` Url string `json:"url,omitempty"` RequestSenderIp string `json:"requestSenderIp,omitempty"` @@ -194,17 +200,8 @@ type DataUnmarshaler interface { } func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error { - bed.Protocol = Protocol{ - Name: entry.ProtocolName, - LongName: entry.ProtocolLongName, - Abbreviation: entry.ProtocolAbbreviation, - Version: entry.ProtocolVersion, - BackgroundColor: entry.ProtocolBackgroundColor, - ForegroundColor: entry.ProtocolForegroundColor, - FontSize: entry.ProtocolFontSize, - ReferenceLink: entry.ProtocolReferenceLink, - } - bed.Id = entry.EntryId + bed.Protocol = entry.Protocol + bed.Id = entry.Id bed.Url = entry.Url bed.RequestSenderIp = entry.RequestSenderIp bed.Service = entry.Service @@ -228,6 +225,21 @@ const ( BODY string = "body" ) +type SectionData struct { + Type string `json:"type"` + Title string `json:"title"` + Data string `json:"data"` + Encoding string `json:"encoding,omitempty"` + MimeType string `json:"mimeType,omitempty"` + Selector string `json:"selector,omitempty"` +} + +type TableData struct { + Name string `json:"name"` + Value interface{} `json:"value"` + Selector string `json:"selector"` +} + const ( TypeHttpRequest = iota TypeHttpResponse diff --git a/tap/extensions/amqp/helpers.go b/tap/extensions/amqp/helpers.go index f7a1842a7..188ae18a1 100644 --- a/tap/extensions/amqp/helpers.go +++ b/tap/extensions/amqp/helpers.go @@ -131,97 +131,109 @@ func representProperties(properties map[string]interface{}, rep []interface{}) ( userId := "" appId := "" - if properties["ContentType"] != nil { - contentType = properties["ContentType"].(string) + if properties["contentType"] != nil { + contentType = properties["contentType"].(string) } - if properties["ContentEncoding"] != nil { - contentEncoding = properties["ContentEncoding"].(string) + if properties["contentEncoding"] != nil { + contentEncoding = properties["contentEncoding"].(string) } - if properties["Delivery Mode"] != nil { - deliveryMode = fmt.Sprintf("%g", properties["DeliveryMode"].(float64)) + if properties["deliveryMode"] != nil { + deliveryMode = fmt.Sprintf("%g", properties["deliveryMode"].(float64)) } - if properties["Priority"] != nil { - priority = fmt.Sprintf("%g", properties["Priority"].(float64)) + if properties["priority"] != nil { + priority = fmt.Sprintf("%g", properties["priority"].(float64)) } - if properties["CorrelationId"] != nil { - correlationId = properties["CorrelationId"].(string) + if properties["correlationId"] != nil { + correlationId = properties["correlationId"].(string) } - if properties["ReplyTo"] != nil { - replyTo = properties["ReplyTo"].(string) + if properties["replyTo"] != nil { + replyTo = properties["replyTo"].(string) } - if properties["Expiration"] != nil { - expiration = properties["Expiration"].(string) + if properties["expiration"] != nil { + expiration = properties["expiration"].(string) } - if properties["MessageId"] != nil { - messageId = properties["MessageId"].(string) + if properties["messageId"] != nil { + messageId = properties["messageId"].(string) } - if properties["Timestamp"] != nil { - timestamp = properties["Timestamp"].(string) + if properties["timestamp"] != nil { + timestamp = properties["timestamp"].(string) } - if properties["Type"] != nil { - _type = properties["Type"].(string) + if properties["type"] != nil { + _type = properties["type"].(string) } - if properties["UserId"] != nil { - userId = properties["UserId"].(string) + if properties["userId"] != nil { + userId = properties["userId"].(string) } - if properties["AppId"] != nil { - appId = properties["AppId"].(string) + if properties["appId"] != nil { + appId = properties["appId"].(string) } - props, _ := json.Marshal([]map[string]string{ + props, _ := json.Marshal([]api.TableData{ { - "name": "Content Type", - "value": contentType, + Name: "Content Type", + Value: contentType, + Selector: `request.properties.contentType`, }, { - "name": "Content Encoding", - "value": contentEncoding, + Name: "Content Encoding", + Value: contentEncoding, + Selector: `request.properties.contentEncoding`, }, { - "name": "Delivery Mode", - "value": deliveryMode, + Name: "Delivery Mode", + Value: deliveryMode, + Selector: `request.properties.deliveryMode`, }, { - "name": "Priority", - "value": priority, + Name: "Priority", + Value: priority, + Selector: `request.properties.priority`, }, { - "name": "Correlation ID", - "value": correlationId, + Name: "Correlation ID", + Value: correlationId, + Selector: `request.properties.correlationId`, }, { - "name": "Reply To", - "value": replyTo, + Name: "Reply To", + Value: replyTo, + Selector: `request.properties.replyTo`, }, { - "name": "Expiration", - "value": expiration, + Name: "Expiration", + Value: expiration, + Selector: `request.properties.expiration`, }, { - "name": "Message ID", - "value": messageId, + Name: "Message ID", + Value: messageId, + Selector: `request.properties.messageId`, }, { - "name": "Timestamp", - "value": timestamp, + Name: "Timestamp", + Value: timestamp, + Selector: `request.properties.timestamp`, }, { - "name": "Type", - "value": _type, + Name: "Type", + Value: _type, + Selector: `request.properties.type`, }, { - "name": "User ID", - "value": userId, + Name: "User ID", + Value: userId, + Selector: `request.properties.userId`, }, { - "name": "App ID", - "value": appId, + Name: "App ID", + Value: appId, + Selector: `request.properties.appId`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Properties", - "data": string(props), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Properties", + Data: string(props), }) return rep, contentType, contentEncoding @@ -230,56 +242,62 @@ func representProperties(properties map[string]interface{}, rep []interface{}) ( func representBasicPublish(event map[string]interface{}) []interface{} { rep := make([]interface{}, 0) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Exchange", - "value": event["Exchange"].(string), + Name: "Exchange", + Value: event["exchange"].(string), + Selector: `request.exchange`, }, { - "name": "Routing Key", - "value": event["RoutingKey"].(string), + Name: "Routing Key", + Value: event["routingKey"].(string), + Selector: `request.routingKey`, }, { - "name": "Mandatory", - "value": strconv.FormatBool(event["Mandatory"].(bool)), + Name: "Mandatory", + Value: strconv.FormatBool(event["mandatory"].(bool)), + Selector: `request.mandatory`, }, { - "name": "Immediate", - "value": strconv.FormatBool(event["Immediate"].(bool)), + Name: "Immediate", + Value: strconv.FormatBool(event["immediate"].(bool)), + Selector: `request.immediate`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - properties := event["Properties"].(map[string]interface{}) + properties := event["properties"].(map[string]interface{}) rep, contentType, _ := representProperties(properties, rep) - if properties["Headers"] != nil { - headers := make([]map[string]string, 0) - for name, value := range properties["Headers"].(map[string]interface{}) { - headers = append(headers, map[string]string{ - "name": name, - "value": value.(string), + if properties["headers"] != nil { + headers := make([]api.TableData, 0) + for name, value := range properties["headers"].(map[string]interface{}) { + headers = append(headers, api.TableData{ + Name: name, + Value: value.(string), + Selector: fmt.Sprintf(`request.properties.headers["%s"]`, name), }) } headersMarshaled, _ := json.Marshal(headers) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Headers", - "data": string(headersMarshaled), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Headers", + Data: string(headersMarshaled), }) } - if event["Body"] != nil { - rep = append(rep, map[string]string{ - "type": api.BODY, - "title": "Body", - "encoding": "base64", - "mime_type": contentType, - "data": event["Body"].(string), + if event["body"] != nil { + rep = append(rep, api.SectionData{ + Type: api.BODY, + Title: "Body", + Encoding: "base64", + MimeType: contentType, + Data: event["body"].(string), + Selector: `request.body`, }) } @@ -293,70 +311,77 @@ func representBasicDeliver(event map[string]interface{}) []interface{} { deliveryTag := "" redelivered := "" - if event["ConsumerTag"] != nil { - consumerTag = event["ConsumerTag"].(string) + if event["consumerTag"] != nil { + consumerTag = event["consumerTag"].(string) } - if event["DeliveryTag"] != nil { - deliveryTag = fmt.Sprintf("%g", event["DeliveryTag"].(float64)) + if event["deliveryTag"] != nil { + deliveryTag = fmt.Sprintf("%g", event["deliveryTag"].(float64)) } - if event["Redelivered"] != nil { - redelivered = strconv.FormatBool(event["Redelivered"].(bool)) + if event["redelivered"] != nil { + redelivered = strconv.FormatBool(event["redelivered"].(bool)) } - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Consumer Tag", - "value": consumerTag, + Name: "Consumer Tag", + Value: consumerTag, + Selector: `request.consumerTag`, }, { - "name": "Delivery Tag", - "value": deliveryTag, + Name: "Delivery Tag", + Value: deliveryTag, + Selector: `request.deliveryTag`, }, { - "name": "Redelivered", - "value": redelivered, + Name: "Redelivered", + Value: redelivered, + Selector: `request.redelivered`, }, { - "name": "Exchange", - "value": event["Exchange"].(string), + Name: "Exchange", + Value: event["exchange"].(string), + Selector: `request.exchange`, }, { - "name": "Routing Key", - "value": event["RoutingKey"].(string), + Name: "Routing Key", + Value: event["routingKey"].(string), + Selector: `request.routingKey`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - properties := event["Properties"].(map[string]interface{}) + properties := event["properties"].(map[string]interface{}) rep, contentType, _ := representProperties(properties, rep) - if properties["Headers"] != nil { - headers := make([]map[string]string, 0) - for name, value := range properties["Headers"].(map[string]interface{}) { - headers = append(headers, map[string]string{ - "name": name, - "value": value.(string), + if properties["headers"] != nil { + headers := make([]api.TableData, 0) + for name, value := range properties["headers"].(map[string]interface{}) { + headers = append(headers, api.TableData{ + Name: name, + Value: value.(string), + Selector: fmt.Sprintf(`request.properties.headers["%s"]`, name), }) } headersMarshaled, _ := json.Marshal(headers) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Headers", - "data": string(headersMarshaled), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Headers", + Data: string(headersMarshaled), }) } - if event["Body"] != nil { - rep = append(rep, map[string]string{ - "type": api.BODY, - "title": "Body", - "encoding": "base64", - "mime_type": contentType, - "data": event["Body"].(string), + if event["body"] != nil { + rep = append(rep, api.SectionData{ + Type: api.BODY, + Title: "Body", + Encoding: "base64", + MimeType: contentType, + Data: event["body"].(string), + Selector: `request.body`, }) } @@ -366,51 +391,58 @@ func representBasicDeliver(event map[string]interface{}) []interface{} { func representQueueDeclare(event map[string]interface{}) []interface{} { rep := make([]interface{}, 0) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Queue", - "value": event["Queue"].(string), + Name: "Queue", + Value: event["queue"].(string), + Selector: `request.queue`, }, { - "name": "Passive", - "value": strconv.FormatBool(event["Passive"].(bool)), + Name: "Passive", + Value: strconv.FormatBool(event["passive"].(bool)), + Selector: `request.queue`, }, { - "name": "Durable", - "value": strconv.FormatBool(event["Durable"].(bool)), + Name: "Durable", + Value: strconv.FormatBool(event["durable"].(bool)), + Selector: `request.durable`, }, { - "name": "Exclusive", - "value": strconv.FormatBool(event["Exclusive"].(bool)), + Name: "Exclusive", + Value: strconv.FormatBool(event["exclusive"].(bool)), + Selector: `request.exclusive`, }, { - "name": "Auto Delete", - "value": strconv.FormatBool(event["AutoDelete"].(bool)), + Name: "Auto Delete", + Value: strconv.FormatBool(event["autoDelete"].(bool)), + Selector: `request.autoDelete`, }, { - "name": "NoWait", - "value": strconv.FormatBool(event["NoWait"].(bool)), + Name: "NoWait", + Value: strconv.FormatBool(event["noWait"].(bool)), + Selector: `request.noWait`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - if event["Arguments"] != nil { - headers := make([]map[string]string, 0) - for name, value := range event["Arguments"].(map[string]interface{}) { - headers = append(headers, map[string]string{ - "name": name, - "value": value.(string), + if event["arguments"] != nil { + headers := make([]api.TableData, 0) + for name, value := range event["arguments"].(map[string]interface{}) { + headers = append(headers, api.TableData{ + Name: name, + Value: value.(string), + Selector: fmt.Sprintf(`request.arguments["%s"]`, name), }) } headersMarshaled, _ := json.Marshal(headers) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Arguments", - "data": string(headersMarshaled), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Arguments", + Data: string(headersMarshaled), }) } @@ -420,55 +452,63 @@ func representQueueDeclare(event map[string]interface{}) []interface{} { func representExchangeDeclare(event map[string]interface{}) []interface{} { rep := make([]interface{}, 0) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Exchange", - "value": event["Exchange"].(string), + Name: "Exchange", + Value: event["exchange"].(string), + Selector: `request.exchange`, }, { - "name": "Type", - "value": event["Type"].(string), + Name: "Type", + Value: event["type"].(string), + Selector: `request.type`, }, { - "name": "Passive", - "value": strconv.FormatBool(event["Passive"].(bool)), + Name: "Passive", + Value: strconv.FormatBool(event["passive"].(bool)), + Selector: `request.passive`, }, { - "name": "Durable", - "value": strconv.FormatBool(event["Durable"].(bool)), + Name: "Durable", + Value: strconv.FormatBool(event["durable"].(bool)), + Selector: `request.durable`, }, { - "name": "Auto Delete", - "value": strconv.FormatBool(event["AutoDelete"].(bool)), + Name: "Auto Delete", + Value: strconv.FormatBool(event["autoDelete"].(bool)), + Selector: `request.autoDelete`, }, { - "name": "Internal", - "value": strconv.FormatBool(event["Internal"].(bool)), + Name: "Internal", + Value: strconv.FormatBool(event["internal"].(bool)), + Selector: `request.internal`, }, { - "name": "NoWait", - "value": strconv.FormatBool(event["NoWait"].(bool)), + Name: "NoWait", + Value: strconv.FormatBool(event["noWait"].(bool)), + Selector: `request.noWait`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - if event["Arguments"] != nil { - headers := make([]map[string]string, 0) - for name, value := range event["Arguments"].(map[string]interface{}) { - headers = append(headers, map[string]string{ - "name": name, - "value": value.(string), + if event["arguments"] != nil { + headers := make([]api.TableData, 0) + for name, value := range event["arguments"].(map[string]interface{}) { + headers = append(headers, api.TableData{ + Name: name, + Value: value.(string), + Selector: fmt.Sprintf(`request.arguments["%s"]`, name), }) } headersMarshaled, _ := json.Marshal(headers) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Arguments", - "data": string(headersMarshaled), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Arguments", + Data: string(headersMarshaled), }) } @@ -478,33 +518,37 @@ func representExchangeDeclare(event map[string]interface{}) []interface{} { func representConnectionStart(event map[string]interface{}) []interface{} { rep := make([]interface{}, 0) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Version Major", - "value": fmt.Sprintf("%g", event["VersionMajor"].(float64)), + Name: "Version Major", + Value: fmt.Sprintf("%g", event["versionMajor"].(float64)), + Selector: `request.versionMajor`, }, { - "name": "Version Minor", - "value": fmt.Sprintf("%g", event["VersionMinor"].(float64)), + Name: "Version Minor", + Value: fmt.Sprintf("%g", event["versionMinor"].(float64)), + Selector: `request.versionMinor`, }, { - "name": "Mechanisms", - "value": event["Mechanisms"].(string), + Name: "Mechanisms", + Value: event["mechanisms"].(string), + Selector: `request.mechanisms`, }, { - "name": "Locales", - "value": event["Locales"].(string), + Name: "Locales", + Value: event["locales"].(string), + Selector: `request.locales`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - if event["ServerProperties"] != nil { - headers := make([]map[string]string, 0) - for name, value := range event["ServerProperties"].(map[string]interface{}) { + if event["serverProperties"] != nil { + headers := make([]api.TableData, 0) + for name, value := range event["serverProperties"].(map[string]interface{}) { var outcome string switch value.(type) { case string: @@ -517,16 +561,17 @@ func representConnectionStart(event map[string]interface{}) []interface{} { default: panic("Unknown data type for the server property!") } - headers = append(headers, map[string]string{ - "name": name, - "value": outcome, + headers = append(headers, api.TableData{ + Name: name, + Value: outcome, + Selector: fmt.Sprintf(`request.serverProperties["%s"]`, name), }) } headersMarshaled, _ := json.Marshal(headers) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Server Properties", - "data": string(headersMarshaled), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Server Properties", + Data: string(headersMarshaled), }) } @@ -536,28 +581,32 @@ func representConnectionStart(event map[string]interface{}) []interface{} { func representConnectionClose(event map[string]interface{}) []interface{} { rep := make([]interface{}, 0) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Reply Code", - "value": fmt.Sprintf("%g", event["ReplyCode"].(float64)), + Name: "Reply Code", + Value: fmt.Sprintf("%g", event["replyCode"].(float64)), + Selector: `request.replyCode`, }, { - "name": "Reply Text", - "value": event["ReplyText"].(string), + Name: "Reply Text", + Value: event["replyText"].(string), + Selector: `request.replyText`, }, { - "name": "Class ID", - "value": fmt.Sprintf("%g", event["ClassId"].(float64)), + Name: "Class ID", + Value: fmt.Sprintf("%g", event["classId"].(float64)), + Selector: `request.classId`, }, { - "name": "Method ID", - "value": fmt.Sprintf("%g", event["MethodId"].(float64)), + Name: "Method ID", + Value: fmt.Sprintf("%g", event["methodId"].(float64)), + Selector: `request.methodId`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) return rep @@ -566,43 +615,48 @@ func representConnectionClose(event map[string]interface{}) []interface{} { func representQueueBind(event map[string]interface{}) []interface{} { rep := make([]interface{}, 0) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Queue", - "value": event["Queue"].(string), + Name: "Queue", + Value: event["queue"].(string), + Selector: `request.queue`, }, { - "name": "Exchange", - "value": event["Exchange"].(string), + Name: "Exchange", + Value: event["exchange"].(string), + Selector: `request.exchange`, }, { - "name": "RoutingKey", - "value": event["RoutingKey"].(string), + Name: "RoutingKey", + Value: event["routingKey"].(string), + Selector: `request.routingKey`, }, { - "name": "NoWait", - "value": strconv.FormatBool(event["NoWait"].(bool)), + Name: "NoWait", + Value: strconv.FormatBool(event["noWait"].(bool)), + Selector: `request.noWait`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - if event["Arguments"] != nil { - headers := make([]map[string]string, 0) - for name, value := range event["Arguments"].(map[string]interface{}) { - headers = append(headers, map[string]string{ - "name": name, - "value": value.(string), + if event["arguments"] != nil { + headers := make([]api.TableData, 0) + for name, value := range event["arguments"].(map[string]interface{}) { + headers = append(headers, api.TableData{ + Name: name, + Value: value.(string), + Selector: fmt.Sprintf(`request.arguments["%s"]`, name), }) } headersMarshaled, _ := json.Marshal(headers) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Arguments", - "data": string(headersMarshaled), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Arguments", + Data: string(headersMarshaled), }) } @@ -612,51 +666,58 @@ func representQueueBind(event map[string]interface{}) []interface{} { func representBasicConsume(event map[string]interface{}) []interface{} { rep := make([]interface{}, 0) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Queue", - "value": event["Queue"].(string), + Name: "Queue", + Value: event["queue"].(string), + Selector: `request.queue`, }, { - "name": "Consumer Tag", - "value": event["ConsumerTag"].(string), + Name: "Consumer Tag", + Value: event["consumerTag"].(string), + Selector: `request.consumerTag`, }, { - "name": "No Local", - "value": strconv.FormatBool(event["NoLocal"].(bool)), + Name: "No Local", + Value: strconv.FormatBool(event["noLocal"].(bool)), + Selector: `request.noLocal`, }, { - "name": "No Ack", - "value": strconv.FormatBool(event["NoAck"].(bool)), + Name: "No Ack", + Value: strconv.FormatBool(event["noAck"].(bool)), + Selector: `request.noAck`, }, { - "name": "Exclusive", - "value": strconv.FormatBool(event["Exclusive"].(bool)), + Name: "Exclusive", + Value: strconv.FormatBool(event["exclusive"].(bool)), + Selector: `request.exclusive`, }, { - "name": "NoWait", - "value": strconv.FormatBool(event["NoWait"].(bool)), + Name: "NoWait", + Value: strconv.FormatBool(event["noWait"].(bool)), + Selector: `request.noWait`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - if event["Arguments"] != nil { - headers := make([]map[string]string, 0) - for name, value := range event["Arguments"].(map[string]interface{}) { - headers = append(headers, map[string]string{ - "name": name, - "value": value.(string), + if event["arguments"] != nil { + headers := make([]api.TableData, 0) + for name, value := range event["arguments"].(map[string]interface{}) { + headers = append(headers, api.TableData{ + Name: name, + Value: value.(string), + Selector: fmt.Sprintf(`request.arguments["%s"]`, name), }) } headersMarshaled, _ := json.Marshal(headers) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Arguments", - "data": string(headersMarshaled), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Arguments", + Data: string(headersMarshaled), }) } diff --git a/tap/extensions/amqp/main.go b/tap/extensions/amqp/main.go index 133e8997a..dc2bbf9bc 100644 --- a/tap/extensions/amqp/main.go +++ b/tap/extensions/amqp/main.go @@ -16,6 +16,7 @@ var protocol api.Protocol = api.Protocol{ Name: "amqp", LongName: "Advanced Message Queuing Protocol 0-9-1", Abbreviation: "AMQP", + Macro: "amqp", Version: "0-9-1", BackgroundColor: "#ff6600", ForegroundColor: "#ffffff", @@ -222,7 +223,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co } } -func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry { +func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry { request := item.Pair.Request.Payload.(map[string]interface{}) reqDetails := request["details"].(map[string]interface{}) service := "amqp" @@ -235,75 +236,79 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve summary := "" switch request["method"] { case basicMethodMap[40]: - summary = reqDetails["Exchange"].(string) + summary = reqDetails["exchange"].(string) break case basicMethodMap[60]: - summary = reqDetails["Exchange"].(string) + summary = reqDetails["exchange"].(string) break case exchangeMethodMap[10]: - summary = reqDetails["Exchange"].(string) + summary = reqDetails["exchange"].(string) break case queueMethodMap[10]: - summary = reqDetails["Queue"].(string) + summary = reqDetails["queue"].(string) break case connectionMethodMap[10]: summary = fmt.Sprintf( "%s.%s", - strconv.Itoa(int(reqDetails["VersionMajor"].(float64))), - strconv.Itoa(int(reqDetails["VersionMinor"].(float64))), + strconv.Itoa(int(reqDetails["versionMajor"].(float64))), + strconv.Itoa(int(reqDetails["versionMinor"].(float64))), ) break case connectionMethodMap[50]: - summary = reqDetails["ReplyText"].(string) + summary = reqDetails["replyText"].(string) break case queueMethodMap[20]: - summary = reqDetails["Queue"].(string) + summary = reqDetails["queue"].(string) break case basicMethodMap[20]: - summary = reqDetails["Queue"].(string) + summary = reqDetails["queue"].(string) break } request["url"] = summary - entryBytes, _ := json.Marshal(item.Pair) + reqDetails["method"] = request["method"] return &api.MizuEntry{ - ProtocolName: protocol.Name, - ProtocolLongName: protocol.LongName, - ProtocolAbbreviation: protocol.Abbreviation, - ProtocolVersion: protocol.Version, - ProtocolBackgroundColor: protocol.BackgroundColor, - ProtocolForegroundColor: protocol.ForegroundColor, - ProtocolFontSize: protocol.FontSize, - ProtocolReferenceLink: protocol.ReferenceLink, - EntryId: entryId, - Entry: string(entryBytes), - Url: fmt.Sprintf("%s%s", service, summary), - Method: request["method"].(string), - Status: 0, - RequestSenderIp: item.ConnectionInfo.ClientIP, - Service: service, - Timestamp: item.Timestamp, - ElapsedTime: 0, - Path: summary, - ResolvedSource: resolvedSource, - ResolvedDestination: resolvedDestination, - SourceIp: item.ConnectionInfo.ClientIP, - DestinationIp: item.ConnectionInfo.ServerIP, - SourcePort: item.ConnectionInfo.ClientPort, - DestinationPort: item.ConnectionInfo.ServerPort, - IsOutgoing: item.ConnectionInfo.IsOutgoing, + Protocol: protocol, + Source: &api.TCP{ + Name: resolvedSource, + IP: item.ConnectionInfo.ClientIP, + Port: item.ConnectionInfo.ClientPort, + }, + Destination: &api.TCP{ + Name: resolvedDestination, + IP: item.ConnectionInfo.ServerIP, + Port: item.ConnectionInfo.ServerPort, + }, + Outgoing: item.ConnectionInfo.IsOutgoing, + Request: reqDetails, + Url: fmt.Sprintf("%s%s", service, summary), + Method: request["method"].(string), + Status: 0, + RequestSenderIp: item.ConnectionInfo.ClientIP, + Service: service, + Timestamp: item.Timestamp, + StartTime: item.Pair.Request.CaptureTime, + ElapsedTime: 0, + Summary: summary, + ResolvedSource: resolvedSource, + ResolvedDestination: resolvedDestination, + SourceIp: item.ConnectionInfo.ClientIP, + DestinationIp: item.ConnectionInfo.ServerIP, + SourcePort: item.ConnectionInfo.ClientPort, + DestinationPort: item.ConnectionInfo.ServerPort, + IsOutgoing: item.ConnectionInfo.IsOutgoing, } } func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { return &api.BaseEntryDetails{ - Id: entry.EntryId, + Id: entry.Id, Protocol: protocol, Url: entry.Url, RequestSenderIp: entry.RequestSenderIp, Service: entry.Service, - Summary: entry.Path, + Summary: entry.Summary, StatusCode: entry.Status, Method: entry.Method, Timestamp: entry.Timestamp, @@ -320,39 +325,35 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { } } -func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) { - p = protocol +func (d dissecting) Represent(protoIn api.Protocol, request map[string]interface{}, response map[string]interface{}) (protoOut api.Protocol, object []byte, bodySize int64, err error) { + protoOut = protocol bodySize = 0 - var root map[string]interface{} - json.Unmarshal([]byte(entry.Entry), &root) representation := make(map[string]interface{}, 0) - request := root["request"].(map[string]interface{})["payload"].(map[string]interface{}) var repRequest []interface{} - details := request["details"].(map[string]interface{}) switch request["method"].(string) { case basicMethodMap[40]: - repRequest = representBasicPublish(details) + repRequest = representBasicPublish(request) break case basicMethodMap[60]: - repRequest = representBasicDeliver(details) + repRequest = representBasicDeliver(request) break case queueMethodMap[10]: - repRequest = representQueueDeclare(details) + repRequest = representQueueDeclare(request) break case exchangeMethodMap[10]: - repRequest = representExchangeDeclare(details) + repRequest = representExchangeDeclare(request) break case connectionMethodMap[10]: - repRequest = representConnectionStart(details) + repRequest = representConnectionStart(request) break case connectionMethodMap[50]: - repRequest = representConnectionClose(details) + repRequest = representConnectionClose(request) break case queueMethodMap[20]: - repRequest = representQueueBind(details) + repRequest = representQueueBind(request) break case basicMethodMap[20]: - repRequest = representBasicConsume(details) + repRequest = representBasicConsume(request) break } representation["request"] = repRequest @@ -360,4 +361,10 @@ func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []by return } +func (d dissecting) Macros() map[string]string { + return map[string]string{ + `amqp`: fmt.Sprintf(`proto.abbr == "%s"`, protocol.Abbreviation), + } +} + var Dissector dissecting diff --git a/tap/extensions/amqp/spec091.go b/tap/extensions/amqp/spec091.go index 3dd3dcbd3..0af77a26c 100644 --- a/tap/extensions/amqp/spec091.go +++ b/tap/extensions/amqp/spec091.go @@ -71,11 +71,11 @@ func isSoftExceptionCode(code int) bool { } type ConnectionStart struct { - VersionMajor byte - VersionMinor byte - ServerProperties Table - Mechanisms string - Locales string + VersionMajor byte `json:"versionMajor"` + VersionMinor byte `json:"versionMinor"` + ServerProperties Table `json:"serverProperties"` + Mechanisms string `json:"mechanisms"` + Locales string `json:"locales"` } func (msg *ConnectionStart) id() (uint16, uint16) { @@ -429,10 +429,10 @@ func (msg *connectionOpenOk) read(r io.Reader) (err error) { } type ConnectionClose struct { - ReplyCode uint16 - ReplyText string - ClassId uint16 - MethodId uint16 + ReplyCode uint16 `json:"relyCode"` + ReplyText string `json:"replyText"` + ClassId uint16 `json:"classId"` + MethodId uint16 `json:"methodId"` } func (msg *ConnectionClose) id() (uint16, uint16) { @@ -767,14 +767,14 @@ func (msg *channelCloseOk) read(r io.Reader) (err error) { type ExchangeDeclare struct { reserved1 uint16 - Exchange string - Type string - Passive bool - Durable bool - AutoDelete bool - Internal bool - NoWait bool - Arguments Table + Exchange string `json:"exchange"` + Type string `json:"type"` + Passive bool `json:"passive"` + Durable bool `json:"durable"` + AutoDelete bool `json:"autoDelete"` + Internal bool `json:"internal"` + NoWait bool `json:"noWait"` + Arguments Table `json:"arguments"` } func (msg *ExchangeDeclare) id() (uint16, uint16) { @@ -1163,13 +1163,13 @@ func (msg *exchangeUnbindOk) read(r io.Reader) (err error) { type QueueDeclare struct { reserved1 uint16 - Queue string - Passive bool - Durable bool - Exclusive bool - AutoDelete bool - NoWait bool - Arguments Table + Queue string `json:"queue"` + Passive bool `json:"passive"` + Durable bool `json:"durable"` + Exclusive bool `json:"exclusive"` + AutoDelete bool `json:"autoDelete"` + NoWait bool `json:"noWait"` + Arguments Table `json:"arguments"` } func (msg *QueueDeclare) id() (uint16, uint16) { @@ -1297,11 +1297,11 @@ func (msg *QueueDeclareOk) read(r io.Reader) (err error) { type QueueBind struct { reserved1 uint16 - Queue string - Exchange string - RoutingKey string - NoWait bool - Arguments Table + Queue string `json:"queue"` + Exchange string `json:"exchange"` + RoutingKey string `json:"routingKey"` + NoWait bool `json:"noWait"` + Arguments Table `json:"arguments"` } func (msg *QueueBind) id() (uint16, uint16) { @@ -1737,13 +1737,13 @@ func (msg *basicQosOk) read(r io.Reader) (err error) { type BasicConsume struct { reserved1 uint16 - Queue string - ConsumerTag string - NoLocal bool - NoAck bool - Exclusive bool - NoWait bool - Arguments Table + Queue string `json:"queue"` + ConsumerTag string `json:"consumerTag"` + NoLocal bool `json:"noLocal"` + NoAck bool `json:"noAck"` + Exclusive bool `json:"exclusive"` + NoWait bool `json:"noWait"` + Arguments Table `json:"arguments"` } func (msg *BasicConsume) id() (uint16, uint16) { @@ -1932,12 +1932,12 @@ func (msg *basicCancelOk) read(r io.Reader) (err error) { type BasicPublish struct { reserved1 uint16 - Exchange string - RoutingKey string - Mandatory bool - Immediate bool - Properties Properties - Body []byte + Exchange string `json:"exchange"` + RoutingKey string `json:"routingKey"` + Mandatory bool `json:"mandatory"` + Immediate bool `json:"immediate"` + Properties Properties `json:"properties"` + Body []byte `json:"body"` } func (msg *BasicPublish) id() (uint16, uint16) { @@ -2072,13 +2072,13 @@ func (msg *basicReturn) read(r io.Reader) (err error) { } type BasicDeliver struct { - ConsumerTag string - DeliveryTag uint64 - Redelivered bool - Exchange string - RoutingKey string - Properties Properties - Body []byte + ConsumerTag string `json:"consumerTag"` + DeliveryTag uint64 `json:"deliveryTag"` + Redelivered bool `json:"redelivered"` + Exchange string `json:"exchange"` + RoutingKey string `json:"routingKey"` + Properties Properties `json:"properties"` + Body []byte `json:"body"` } func (msg *BasicDeliver) id() (uint16, uint16) { diff --git a/tap/extensions/amqp/types.go b/tap/extensions/amqp/types.go index ea57556fa..1adee51c6 100644 --- a/tap/extensions/amqp/types.go +++ b/tap/extensions/amqp/types.go @@ -93,19 +93,19 @@ func (e Error) Error() string { // Used by header frames to capture routing and header information type Properties struct { - ContentType string // MIME content type - ContentEncoding string // MIME content encoding - Headers Table // Application or header exchange table - DeliveryMode uint8 // queue implementation use - Transient (1) or Persistent (2) - Priority uint8 // queue implementation use - 0 to 9 - CorrelationId string // application use - correlation identifier - ReplyTo string // application use - address to to reply to (ex: RPC) - Expiration string // implementation use - message expiration spec - MessageId string // application use - message identifier - Timestamp time.Time // application use - message timestamp - Type string // application use - message type name - UserId string // application use - creating user id - AppId string // application use - creating application + ContentType string `json:"contentType"` // MIME content type + ContentEncoding string `json:"contentEncoding"` // MIME content encoding + Headers Table `json:"headers"` // Application or header exchange table + DeliveryMode uint8 `json:"deliveryMode"` // queue implementation use - Transient (1) or Persistent (2) + Priority uint8 `json:"priority"` // queue implementation use - 0 to 9 + CorrelationId string `json:"correlationId"` // application use - correlation identifier + ReplyTo string `json:"replyTo"` // application use - address to to reply to (ex: RPC) + Expiration string `json:"expiration"` // implementation use - message expiration spec + MessageId string `json:"messageId"` // application use - message identifier + Timestamp time.Time `json:"timestamp"` // application use - message timestamp + Type string `json:"type"` // application use - message type name + UserId string `json:"userId"` // application use - creating user id + AppId string `json:"appId"` // application use - creating application reserved1 string // was cluster-id - process for buffer consumption } diff --git a/tap/extensions/http/helpers.go b/tap/extensions/http/helpers.go new file mode 100644 index 000000000..9348ea3cd --- /dev/null +++ b/tap/extensions/http/helpers.go @@ -0,0 +1,35 @@ +package main + +import ( + "encoding/json" + "fmt" + + "github.com/up9inc/mizu/tap/api" +) + +func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}) { + newMap = make(map[string]interface{}) + for _, header := range mapSlice { + h := header.(map[string]interface{}) + newMap[h["name"].(string)] = h["value"] + } + + return +} + +func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (representation string) { + var table []api.TableData + for _, header := range mapSlice { + h := header.(map[string]interface{}) + selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, h["name"].(string)) + table = append(table, api.TableData{ + Name: h["name"].(string), + Value: h["value"], + Selector: selector, + }) + } + + obj, _ := json.Marshal(table) + representation = string(obj) + return +} diff --git a/tap/extensions/http/main.go b/tap/extensions/http/main.go index 3ca432a71..b58fc7a02 100644 --- a/tap/extensions/http/main.go +++ b/tap/extensions/http/main.go @@ -17,6 +17,7 @@ var protocol api.Protocol = api.Protocol{ Name: "http", LongName: "Hypertext Transfer Protocol -- HTTP/1.1", Abbreviation: "HTTP", + Macro: "http", Version: "1.1", BackgroundColor: "#205cf5", ForegroundColor: "#ffffff", @@ -30,6 +31,7 @@ var http2Protocol api.Protocol = api.Protocol{ Name: "http", LongName: "Hypertext Transfer Protocol Version 2 (HTTP/2) (gRPC)", Abbreviation: "HTTP/2", + Macro: "grpc", Version: "2.0", BackgroundColor: "#244c5a", ForegroundColor: "#ffffff", @@ -117,7 +119,7 @@ func SetHostname(address, newHostname string) string { return replacedUrl.String() } -func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry { +func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry { var host, scheme, authority, path, service string request := item.Pair.Request.Payload.(map[string]interface{}) @@ -145,10 +147,32 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve service = fmt.Sprintf("%s://%s", scheme, authority) } else { service = fmt.Sprintf("http://%s", host) - path = reqDetails["url"].(string) + u, err := url.Parse(reqDetails["url"].(string)) + if err != nil { + path = reqDetails["url"].(string) + } else { + path = u.Path + } } - request["url"] = path + request["url"] = reqDetails["url"].(string) + reqDetails["path"] = path + reqDetails["summary"] = path + + // Rearrange the maps for the querying + reqDetails["_headers"] = reqDetails["headers"] + reqDetails["headers"] = mapSliceRebuildAsMap(reqDetails["_headers"].([]interface{})) + resDetails["_headers"] = resDetails["headers"] + resDetails["headers"] = mapSliceRebuildAsMap(resDetails["_headers"].([]interface{})) + + reqDetails["_cookies"] = reqDetails["cookies"] + reqDetails["cookies"] = mapSliceRebuildAsMap(reqDetails["_cookies"].([]interface{})) + resDetails["_cookies"] = resDetails["cookies"] + resDetails["cookies"] = mapSliceRebuildAsMap(resDetails["_cookies"].([]interface{})) + + reqDetails["_queryString"] = reqDetails["queryString"] + reqDetails["queryString"] = mapSliceRebuildAsMap(reqDetails["_queryString"].([]interface{})) + if resolvedDestination != "" { service = SetHostname(service, resolvedDestination) } else if resolvedSource != "" { @@ -156,51 +180,59 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve } elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds() - entryBytes, _ := json.Marshal(item.Pair) + httpPair, _ := json.Marshal(item.Pair) + _protocol := protocol + _protocol.Version = item.Protocol.Version return &api.MizuEntry{ - ProtocolName: protocol.Name, - ProtocolLongName: protocol.LongName, - ProtocolAbbreviation: protocol.Abbreviation, - ProtocolVersion: item.Protocol.Version, - ProtocolBackgroundColor: protocol.BackgroundColor, - ProtocolForegroundColor: protocol.ForegroundColor, - ProtocolFontSize: protocol.FontSize, - ProtocolReferenceLink: protocol.ReferenceLink, - EntryId: entryId, - Entry: string(entryBytes), - Url: fmt.Sprintf("%s%s", service, path), - Method: reqDetails["method"].(string), - Status: int(resDetails["status"].(float64)), - RequestSenderIp: item.ConnectionInfo.ClientIP, - Service: service, - Timestamp: item.Timestamp, - ElapsedTime: elapsedTime, - Path: path, - ResolvedSource: resolvedSource, - ResolvedDestination: resolvedDestination, - SourceIp: item.ConnectionInfo.ClientIP, - DestinationIp: item.ConnectionInfo.ServerIP, - SourcePort: item.ConnectionInfo.ClientPort, - DestinationPort: item.ConnectionInfo.ServerPort, - IsOutgoing: item.ConnectionInfo.IsOutgoing, + Protocol: _protocol, + Source: &api.TCP{ + Name: resolvedSource, + IP: item.ConnectionInfo.ClientIP, + Port: item.ConnectionInfo.ClientPort, + }, + Destination: &api.TCP{ + Name: resolvedDestination, + IP: item.ConnectionInfo.ServerIP, + Port: item.ConnectionInfo.ServerPort, + }, + Outgoing: item.ConnectionInfo.IsOutgoing, + Request: reqDetails, + Response: resDetails, + Url: fmt.Sprintf("%s%s", service, path), + Method: reqDetails["method"].(string), + Status: int(resDetails["status"].(float64)), + RequestSenderIp: item.ConnectionInfo.ClientIP, + Service: service, + Timestamp: item.Timestamp, + StartTime: item.Pair.Request.CaptureTime, + ElapsedTime: elapsedTime, + Summary: path, + ResolvedSource: resolvedSource, + ResolvedDestination: resolvedDestination, + SourceIp: item.ConnectionInfo.ClientIP, + DestinationIp: item.ConnectionInfo.ServerIP, + SourcePort: item.ConnectionInfo.ClientPort, + DestinationPort: item.ConnectionInfo.ServerPort, + IsOutgoing: item.ConnectionInfo.IsOutgoing, + HTTPPair: string(httpPair), } } func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { var p api.Protocol - if entry.ProtocolVersion == "2.0" { + if entry.Protocol.Version == "2.0" { p = http2Protocol } else { p = protocol } return &api.BaseEntryDetails{ - Id: entry.EntryId, + Id: entry.Id, Protocol: p, Url: entry.Url, RequestSenderIp: entry.RequestSenderIp, Service: entry.Service, Path: entry.Path, - Summary: entry.Path, + Summary: entry.Summary, StatusCode: entry.Status, Method: entry.Method, Timestamp: entry.Timestamp, @@ -218,45 +250,50 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { } func representRequest(request map[string]interface{}) (repRequest []interface{}) { - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Method", - "value": request["method"].(string), + Name: "Method", + Value: request["method"].(string), + Selector: `request.method`, }, { - "name": "URL", - "value": request["url"].(string), + Name: "URL", + Value: request["url"].(string), + Selector: `request.url`, }, { - "name": "Body Size", - "value": fmt.Sprintf("%g bytes", request["bodySize"].(float64)), + Name: "Path", + Value: request["path"].(string), + Selector: `request.path`, + }, + { + Name: "Body Size (bytes)", + Value: int64(request["bodySize"].(float64)), + Selector: `request.bodySize`, }, }) - repRequest = append(repRequest, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + repRequest = append(repRequest, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - headers, _ := json.Marshal(request["headers"].([]interface{})) - repRequest = append(repRequest, map[string]string{ - "type": api.TABLE, - "title": "Headers", - "data": string(headers), + repRequest = append(repRequest, api.SectionData{ + Type: api.TABLE, + Title: "Headers", + Data: representMapSliceAsTable(request["_headers"].([]interface{}), `request.headers`), }) - cookies, _ := json.Marshal(request["cookies"].([]interface{})) - repRequest = append(repRequest, map[string]string{ - "type": api.TABLE, - "title": "Cookies", - "data": string(cookies), + repRequest = append(repRequest, api.SectionData{ + Type: api.TABLE, + Title: "Cookies", + Data: representMapSliceAsTable(request["_cookies"].([]interface{}), `request.cookies`), }) - queryString, _ := json.Marshal(request["queryString"].([]interface{})) - repRequest = append(repRequest, map[string]string{ - "type": api.TABLE, - "title": "Query String", - "data": string(queryString), + repRequest = append(repRequest, api.SectionData{ + Type: api.TABLE, + Title: "Query String", + Data: representMapSliceAsTable(request["_queryString"].([]interface{}), `request.queryString`), }) postData, _ := request["postData"].(map[string]interface{}) @@ -266,12 +303,12 @@ func representRequest(request map[string]interface{}) (repRequest []interface{}) } text, _ := postData["text"] if text != nil { - repRequest = append(repRequest, map[string]string{ - "type": api.BODY, - "title": "POST Data (text/plain)", - "encoding": "", - "mime_type": mimeType.(string), - "data": text.(string), + repRequest = append(repRequest, api.SectionData{ + Type: api.BODY, + Title: "POST Data (text/plain)", + MimeType: mimeType.(string), + Data: text.(string), + Selector: `request.postData.text`, }) } @@ -285,16 +322,16 @@ func representRequest(request map[string]interface{}) (repRequest []interface{}) "value": string(params), }, }) - repRequest = append(repRequest, map[string]string{ - "type": api.TABLE, - "title": "POST Data (multipart/form-data)", - "data": string(multipart), + repRequest = append(repRequest, api.SectionData{ + Type: api.TABLE, + Title: "POST Data (multipart/form-data)", + Data: string(multipart), }) } else { - repRequest = append(repRequest, map[string]string{ - "type": api.TABLE, - "title": "POST Data (application/x-www-form-urlencoded)", - "data": string(params), + repRequest = append(repRequest, api.SectionData{ + Type: api.TABLE, + Title: "POST Data (application/x-www-form-urlencoded)", + Data: representMapSliceAsTable(postData["params"].([]interface{}), `request.postData.params`), }) } } @@ -308,38 +345,39 @@ func representResponse(response map[string]interface{}) (repResponse []interface bodySize = int64(response["bodySize"].(float64)) - details, _ := json.Marshal([]map[string]string{ + details, _ := json.Marshal([]api.TableData{ { - "name": "Status", - "value": fmt.Sprintf("%g", response["status"].(float64)), + Name: "Status", + Value: int64(response["status"].(float64)), + Selector: `response.status`, }, { - "name": "Status Text", - "value": response["statusText"].(string), + Name: "Status Text", + Value: response["statusText"].(string), + Selector: `response.statusText`, }, { - "name": "Body Size", - "value": fmt.Sprintf("%d bytes", bodySize), + Name: "Body Size (bytes)", + Value: bodySize, + Selector: `response.bodySize`, }, }) - repResponse = append(repResponse, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + repResponse = append(repResponse, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) - headers, _ := json.Marshal(response["headers"].([]interface{})) - repResponse = append(repResponse, map[string]string{ - "type": api.TABLE, - "title": "Headers", - "data": string(headers), + repResponse = append(repResponse, api.SectionData{ + Type: api.TABLE, + Title: "Headers", + Data: representMapSliceAsTable(response["_headers"].([]interface{}), `response.headers`), }) - cookies, _ := json.Marshal(response["cookies"].([]interface{})) - repResponse = append(repResponse, map[string]string{ - "type": api.TABLE, - "title": "Cookies", - "data": string(cookies), + repResponse = append(repResponse, api.SectionData{ + Type: api.TABLE, + Title: "Cookies", + Data: representMapSliceAsTable(response["_cookies"].([]interface{}), `response.cookies`), }) content, _ := response["content"].(map[string]interface{}) @@ -350,37 +388,40 @@ func representResponse(response map[string]interface{}) (repResponse []interface encoding, _ := content["encoding"] text, _ := content["text"] if text != nil { - repResponse = append(repResponse, map[string]string{ - "type": api.BODY, - "title": "Body", - "encoding": encoding.(string), - "mime_type": mimeType.(string), - "data": text.(string), + repResponse = append(repResponse, api.SectionData{ + Type: api.BODY, + Title: "Body", + Encoding: encoding.(string), + MimeType: mimeType.(string), + Data: text.(string), + Selector: `response.content.text`, }) } return } -func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) { - if entry.ProtocolVersion == "2.0" { - p = http2Protocol +func (d dissecting) Represent(protoIn api.Protocol, request map[string]interface{}, response map[string]interface{}) (protoOut api.Protocol, object []byte, bodySize int64, err error) { + if protoIn.Version == "2.0" { + protoOut = http2Protocol } else { - p = protocol + protoOut = protocol } - var root map[string]interface{} - json.Unmarshal([]byte(entry.Entry), &root) representation := make(map[string]interface{}, 0) - request := root["request"].(map[string]interface{})["payload"].(map[string]interface{}) - response := root["response"].(map[string]interface{})["payload"].(map[string]interface{}) - reqDetails := request["details"].(map[string]interface{}) - resDetails := response["details"].(map[string]interface{}) - repRequest := representRequest(reqDetails) - repResponse, bodySize := representResponse(resDetails) + repRequest := representRequest(request) + repResponse, bodySize := representResponse(response) representation["request"] = repRequest representation["response"] = repResponse object, err = json.Marshal(representation) return } +func (d dissecting) Macros() map[string]string { + return map[string]string{ + `http`: fmt.Sprintf(`proto.abbr == "%s"`, protocol.Abbreviation), + `grpc`: fmt.Sprintf(`proto.abbr == "%s" and proto.version == "%s"`, protocol.Abbreviation, http2Protocol.Version), + `http2`: fmt.Sprintf(`proto.abbr == "%s" and proto.version == "%s"`, protocol.Abbreviation, http2Protocol.Version), + } +} + var Dissector dissecting diff --git a/tap/extensions/kafka/helpers.go b/tap/extensions/kafka/helpers.go index 3a9a238eb..dffe30bd8 100644 --- a/tap/extensions/kafka/helpers.go +++ b/tap/extensions/kafka/helpers.go @@ -27,48 +27,54 @@ type KafkaWrapper struct { } func representRequestHeader(data map[string]interface{}, rep []interface{}) []interface{} { - requestHeader, _ := json.Marshal([]map[string]string{ + requestHeader, _ := json.Marshal([]api.TableData{ { - "name": "ApiKey", - "value": apiNames[int(data["ApiKey"].(float64))], + Name: "ApiKey", + Value: apiNames[int(data["apiKey"].(float64))], + Selector: `request.apiKey`, }, { - "name": "ApiVersion", - "value": fmt.Sprintf("%d", int(data["ApiVersion"].(float64))), + Name: "ApiVersion", + Value: fmt.Sprintf("%d", int(data["apiVersion"].(float64))), + Selector: `request.apiVersion`, }, { - "name": "Client ID", - "value": data["ClientID"].(string), + Name: "Client ID", + Value: data["clientID"].(string), + Selector: `request.clientID`, }, { - "name": "Correlation ID", - "value": fmt.Sprintf("%d", int(data["CorrelationID"].(float64))), + Name: "Correlation ID", + Value: fmt.Sprintf("%d", int(data["correlationID"].(float64))), + Selector: `request.correlationID`, }, { - "name": "Size", - "value": fmt.Sprintf("%d", int(data["Size"].(float64))), + Name: "Size", + Value: fmt.Sprintf("%d", int(data["size"].(float64))), + Selector: `request.size`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Request Header", - "data": string(requestHeader), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Request Header", + Data: string(requestHeader), }) return rep } func representResponseHeader(data map[string]interface{}, rep []interface{}) []interface{} { - requestHeader, _ := json.Marshal([]map[string]string{ + requestHeader, _ := json.Marshal([]api.TableData{ { - "name": "Correlation ID", - "value": fmt.Sprintf("%d", int(data["CorrelationID"].(float64))), + Name: "Correlation ID", + Value: fmt.Sprintf("%d", int(data["correlationID"].(float64))), + Selector: `response.correlationID`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Response Header", - "data": string(requestHeader), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Response Header", + Data: string(requestHeader), }) return rep @@ -79,46 +85,50 @@ func representMetadataRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) topics := "" allowAutoTopicCreation := "" includeClusterAuthorizedOperations := "" includeTopicAuthorizedOperations := "" - if payload["Topics"] != nil { - x, _ := json.Marshal(payload["Topics"].([]interface{})) + if payload["topics"] != nil { + x, _ := json.Marshal(payload["topics"].([]interface{})) topics = string(x) } - if payload["AllowAutoTopicCreation"] != nil { - allowAutoTopicCreation = strconv.FormatBool(payload["AllowAutoTopicCreation"].(bool)) + if payload["allowAutoTopicCreation"] != nil { + allowAutoTopicCreation = strconv.FormatBool(payload["allowAutoTopicCreation"].(bool)) } - if payload["IncludeClusterAuthorizedOperations"] != nil { - includeClusterAuthorizedOperations = strconv.FormatBool(payload["IncludeClusterAuthorizedOperations"].(bool)) + if payload["includeClusterAuthorizedOperations"] != nil { + includeClusterAuthorizedOperations = strconv.FormatBool(payload["includeClusterAuthorizedOperations"].(bool)) } - if payload["IncludeTopicAuthorizedOperations"] != nil { - includeTopicAuthorizedOperations = strconv.FormatBool(payload["IncludeTopicAuthorizedOperations"].(bool)) + if payload["includeTopicAuthorizedOperations"] != nil { + includeTopicAuthorizedOperations = strconv.FormatBool(payload["includeTopicAuthorizedOperations"].(bool)) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Topics", - "value": topics, + Name: "Topics", + Value: topics, + Selector: `request.payload.topics`, }, { - "name": "Allow Auto Topic Creation", - "value": allowAutoTopicCreation, + Name: "Allow Auto Topic Creation", + Value: allowAutoTopicCreation, + Selector: `request.payload.allowAutoTopicCreation`, }, { - "name": "Include Cluster Authorized Operations", - "value": includeClusterAuthorizedOperations, + Name: "Include Cluster Authorized Operations", + Value: includeClusterAuthorizedOperations, + Selector: `request.payload.includeClusterAuthorizedOperations`, }, { - "name": "Include Topic Authorized Operations", - "value": includeTopicAuthorizedOperations, + Name: "Include Topic Authorized Operations", + Value: includeTopicAuthorizedOperations, + Selector: `request.payload.includeTopicAuthorizedOperations`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -129,63 +139,69 @@ func representMetadataResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) topics := "" - if payload["Topics"] != nil { - _topics, _ := json.Marshal(payload["Topics"].([]interface{})) + if payload["topics"] != nil { + _topics, _ := json.Marshal(payload["topics"].([]interface{})) topics = string(_topics) } brokers := "" - if payload["Brokers"] != nil { - _brokers, _ := json.Marshal(payload["Brokers"].([]interface{})) + if payload["brokers"] != nil { + _brokers, _ := json.Marshal(payload["brokers"].([]interface{})) brokers = string(_brokers) } controllerID := "" clusterID := "" throttleTimeMs := "" clusterAuthorizedOperations := "" - if payload["ControllerID"] != nil { - controllerID = fmt.Sprintf("%d", int(payload["ControllerID"].(float64))) + if payload["controllerID"] != nil { + controllerID = fmt.Sprintf("%d", int(payload["controllerID"].(float64))) } - if payload["ClusterID"] != nil { - clusterID = payload["ClusterID"].(string) + if payload["clusterID"] != nil { + clusterID = payload["clusterID"].(string) } - if payload["ThrottleTimeMs"] != nil { - throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64))) + if payload["throttleTimeMs"] != nil { + throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) } - if payload["ClusterAuthorizedOperations"] != nil { - clusterAuthorizedOperations = fmt.Sprintf("%d", int(payload["ClusterAuthorizedOperations"].(float64))) + if payload["clusterAuthorizedOperations"] != nil { + clusterAuthorizedOperations = fmt.Sprintf("%d", int(payload["clusterAuthorizedOperations"].(float64))) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Throttle Time (ms)", - "value": throttleTimeMs, + Name: "Throttle Time (ms)", + Value: throttleTimeMs, + Selector: `response.payload.throttleTimeMs`, }, { - "name": "Brokers", - "value": brokers, + Name: "Brokers", + Value: brokers, + Selector: `response.payload.brokers`, }, { - "name": "Cluster ID", - "value": clusterID, + Name: "Cluster ID", + Value: clusterID, + Selector: `response.payload.clusterID`, }, { - "name": "Controller ID", - "value": controllerID, + Name: "Controller ID", + Value: controllerID, + Selector: `response.payload.controllerID`, }, { - "name": "Topics", - "value": topics, + Name: "Topics", + Value: topics, + Selector: `response.payload.topics`, }, { - "name": "Cluster Authorized Operations", - "value": clusterAuthorizedOperations, + Name: "Cluster Authorized Operations", + Value: clusterAuthorizedOperations, + Selector: `response.payload.clusterAuthorizedOperations`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -196,29 +212,31 @@ func representApiVersionsRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) clientSoftwareName := "" clientSoftwareVersion := "" - if payload["ClientSoftwareName"] != nil { - clientSoftwareName = payload["ClientSoftwareName"].(string) + if payload["clientSoftwareName"] != nil { + clientSoftwareName = payload["clientSoftwareName"].(string) } - if payload["ClientSoftwareVersion"] != nil { - clientSoftwareVersion = payload["ClientSoftwareVersion"].(string) + if payload["clientSoftwareVersion"] != nil { + clientSoftwareVersion = payload["clientSoftwareVersion"].(string) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Client Software Name", - "value": clientSoftwareName, + Name: "Client Software Name", + Value: clientSoftwareName, + Selector: `request.payload.clientSoftwareName`, }, { - "name": "Client Software Version", - "value": clientSoftwareVersion, + Name: "Client Software Version", + Value: clientSoftwareVersion, + Selector: `request.payload.clientSoftwareVersion`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -229,34 +247,37 @@ func representApiVersionsResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) apiKeys := "" - if payload["TopicNames"] != nil { - x, _ := json.Marshal(payload["ApiKeys"].([]interface{})) + if payload["apiKeys"] != nil { + x, _ := json.Marshal(payload["apiKeys"].([]interface{})) apiKeys = string(x) } throttleTimeMs := "" - if payload["ThrottleTimeMs"] != nil { - throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64))) + if payload["throttleTimeMs"] != nil { + throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Error Code", - "value": fmt.Sprintf("%d", int(payload["ErrorCode"].(float64))), + Name: "Error Code", + Value: fmt.Sprintf("%d", int(payload["errorCode"].(float64))), + Selector: `response.payload.errorCode`, }, { - "name": "ApiKeys", - "value": apiKeys, + Name: "ApiKeys", + Value: apiKeys, + Selector: `response.payload.apiKeys`, }, { - "name": "Throttle Time (ms)", - "value": throttleTimeMs, + Name: "Throttle Time (ms)", + Value: throttleTimeMs, + Selector: `response.payload.throttleTimeMs`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -267,39 +288,43 @@ func representProduceRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) topicData := "" - _topicData := payload["TopicData"] + _topicData := payload["topicData"] if _topicData != nil { x, _ := json.Marshal(_topicData.([]interface{})) topicData = string(x) } transactionalID := "" - if payload["TransactionalID"] != nil { - transactionalID = payload["TransactionalID"].(string) + if payload["transactionalID"] != nil { + transactionalID = payload["transactionalID"].(string) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Transactional ID", - "value": transactionalID, + Name: "Transactional ID", + Value: transactionalID, + Selector: `request.payload.transactionalID`, }, { - "name": "Required Acknowledgements", - "value": fmt.Sprintf("%d", int(payload["RequiredAcks"].(float64))), + Name: "Required Acknowledgements", + Value: fmt.Sprintf("%d", int(payload["requiredAcks"].(float64))), + Selector: `request.payload.requiredAcks`, }, { - "name": "Timeout", - "value": fmt.Sprintf("%d", int(payload["Timeout"].(float64))), + Name: "Timeout", + Value: fmt.Sprintf("%d", int(payload["timeout"].(float64))), + Selector: `request.payload.timeout`, }, { - "name": "Topic Data", - "value": topicData, + Name: "Topic Data", + Value: topicData, + Selector: `request.payload.topicData`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -310,30 +335,32 @@ func representProduceResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) responses := "" - if payload["Responses"] != nil { - _responses, _ := json.Marshal(payload["Responses"].([]interface{})) + if payload["responses"] != nil { + _responses, _ := json.Marshal(payload["responses"].([]interface{})) responses = string(_responses) } throttleTimeMs := "" - if payload["ThrottleTimeMs"] != nil { - throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64))) + if payload["throttleTimeMs"] != nil { + throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Responses", - "value": string(responses), + Name: "Responses", + Value: string(responses), + Selector: `response.payload.responses`, }, { - "name": "Throttle Time (ms)", - "value": throttleTimeMs, + Name: "Throttle Time (ms)", + Value: throttleTimeMs, + Selector: `response.payload.throttleTimeMs`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -344,87 +371,97 @@ func representFetchRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) topics := "" - if payload["Topics"] != nil { - _topics, _ := json.Marshal(payload["Topics"].([]interface{})) + if payload["topics"] != nil { + _topics, _ := json.Marshal(payload["topics"].([]interface{})) topics = string(_topics) } replicaId := "" - if payload["ReplicaId"] != nil { - replicaId = fmt.Sprintf("%d", int(payload["ReplicaId"].(float64))) + if payload["replicaId"] != nil { + replicaId = fmt.Sprintf("%d", int(payload["replicaId"].(float64))) } maxBytes := "" - if payload["MaxBytes"] != nil { - maxBytes = fmt.Sprintf("%d", int(payload["MaxBytes"].(float64))) + if payload["maxBytes"] != nil { + maxBytes = fmt.Sprintf("%d", int(payload["maxBytes"].(float64))) } isolationLevel := "" - if payload["IsolationLevel"] != nil { - isolationLevel = fmt.Sprintf("%d", int(payload["IsolationLevel"].(float64))) + if payload["isolationLevel"] != nil { + isolationLevel = fmt.Sprintf("%d", int(payload["isolationLevel"].(float64))) } sessionId := "" - if payload["SessionId"] != nil { - sessionId = fmt.Sprintf("%d", int(payload["SessionId"].(float64))) + if payload["sessionId"] != nil { + sessionId = fmt.Sprintf("%d", int(payload["sessionId"].(float64))) } sessionEpoch := "" - if payload["SessionEpoch"] != nil { - sessionEpoch = fmt.Sprintf("%d", int(payload["SessionEpoch"].(float64))) + if payload["sessionEpoch"] != nil { + sessionEpoch = fmt.Sprintf("%d", int(payload["sessionEpoch"].(float64))) } forgottenTopicsData := "" - if payload["ForgottenTopicsData"] != nil { - x, _ := json.Marshal(payload["ForgottenTopicsData"].(map[string]interface{})) + if payload["forgottenTopicsData"] != nil { + x, _ := json.Marshal(payload["forgottenTopicsData"].(map[string]interface{})) forgottenTopicsData = string(x) } rackId := "" - if payload["RackId"] != nil { - rackId = payload["RackId"].(string) + if payload["rackId"] != nil { + rackId = payload["rackId"].(string) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Replica ID", - "value": replicaId, + Name: "Replica ID", + Value: replicaId, + Selector: `request.payload.replicaId`, }, { - "name": "Maximum Wait (ms)", - "value": fmt.Sprintf("%d", int(payload["MaxWaitMs"].(float64))), + Name: "Maximum Wait (ms)", + Value: fmt.Sprintf("%d", int(payload["maxWaitMs"].(float64))), + Selector: `request.payload.maxWaitMs`, }, { - "name": "Minimum Bytes", - "value": fmt.Sprintf("%d", int(payload["MinBytes"].(float64))), + Name: "Minimum Bytes", + Value: fmt.Sprintf("%d", int(payload["minBytes"].(float64))), + Selector: `request.payload.minBytes`, }, { - "name": "Maximum Bytes", - "value": maxBytes, + Name: "Maximum Bytes", + Value: maxBytes, + Selector: `request.payload.maxBytes`, }, { - "name": "Isolation Level", - "value": isolationLevel, + Name: "Isolation Level", + Value: isolationLevel, + Selector: `request.payload.isolationLevel`, }, { - "name": "Session ID", - "value": sessionId, + Name: "Session ID", + Value: sessionId, + Selector: `request.payload.sessionId`, }, { - "name": "Session Epoch", - "value": sessionEpoch, + Name: "Session Epoch", + Value: sessionEpoch, + Selector: `request.payload.sessionEpoch`, }, { - "name": "Topics", - "value": topics, + Name: "Topics", + Value: topics, + Selector: `request.payload.topics`, }, { - "name": "Forgotten Topics Data", - "value": forgottenTopicsData, + Name: "Forgotten Topics Data", + Value: forgottenTopicsData, + Selector: `request.payload.forgottenTopicsData`, }, { - "name": "Rack ID", - "value": rackId, + Name: "Rack ID", + Value: rackId, + Selector: `request.payload.rackId`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -435,46 +472,50 @@ func representFetchResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) responses := "" - if payload["Responses"] != nil { - _responses, _ := json.Marshal(payload["Responses"].([]interface{})) + if payload["responses"] != nil { + _responses, _ := json.Marshal(payload["responses"].([]interface{})) responses = string(_responses) } throttleTimeMs := "" - if payload["ThrottleTimeMs"] != nil { - throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64))) + if payload["throttleTimeMs"] != nil { + throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) } errorCode := "" - if payload["ErrorCode"] != nil { - errorCode = fmt.Sprintf("%d", int(payload["ErrorCode"].(float64))) + if payload["errorCode"] != nil { + errorCode = fmt.Sprintf("%d", int(payload["errorCode"].(float64))) } sessionId := "" - if payload["SessionId"] != nil { - sessionId = fmt.Sprintf("%d", int(payload["SessionId"].(float64))) + if payload["sessionId"] != nil { + sessionId = fmt.Sprintf("%d", int(payload["sessionId"].(float64))) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Throttle Time (ms)", - "value": throttleTimeMs, + Name: "Throttle Time (ms)", + Value: throttleTimeMs, + Selector: `response.payload.throttleTimeMs`, }, { - "name": "Error Code", - "value": errorCode, + Name: "Error Code", + Value: errorCode, + Selector: `response.payload.errorCode`, }, { - "name": "Session ID", - "value": sessionId, + Name: "Session ID", + Value: sessionId, + Selector: `response.payload.sessionId`, }, { - "name": "Responses", - "value": responses, + Name: "Responses", + Value: responses, + Selector: `response.payload.responses`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -485,26 +526,28 @@ func representListOffsetsRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) topics := "" - if payload["Topics"] != nil { - _topics, _ := json.Marshal(payload["Topics"].([]interface{})) + if payload["topics"] != nil { + _topics, _ := json.Marshal(payload["topics"].([]interface{})) topics = string(_topics) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Replica ID", - "value": fmt.Sprintf("%d", int(payload["ReplicaId"].(float64))), + Name: "Replica ID", + Value: fmt.Sprintf("%d", int(payload["replicaId"].(float64))), + Selector: `request.payload.replicaId`, }, { - "name": "Topics", - "value": topics, + Name: "Topics", + Value: topics, + Selector: `request.payload.topics`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -515,26 +558,28 @@ func representListOffsetsResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) - topics, _ := json.Marshal(payload["Topics"].([]interface{})) + payload := data["payload"].(map[string]interface{}) + topics, _ := json.Marshal(payload["topics"].([]interface{})) throttleTimeMs := "" - if payload["ThrottleTimeMs"] != nil { - throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64))) + if payload["throttleTimeMs"] != nil { + throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Throttle Time (ms)", - "value": throttleTimeMs, + Name: "Throttle Time (ms)", + Value: throttleTimeMs, + Selector: `response.payload.throttleTimeMs`, }, { - "name": "Topics", - "value": string(topics), + Name: "Topics", + Value: string(topics), + Selector: `response.payload.topics`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -545,30 +590,33 @@ func representCreateTopicsRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) - topics, _ := json.Marshal(payload["Topics"].([]interface{})) + payload := data["payload"].(map[string]interface{}) + topics, _ := json.Marshal(payload["topics"].([]interface{})) validateOnly := "" - if payload["ValidateOnly"] != nil { - validateOnly = strconv.FormatBool(payload["ValidateOnly"].(bool)) + if payload["validateOnly"] != nil { + validateOnly = strconv.FormatBool(payload["validateOnly"].(bool)) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Topics", - "value": string(topics), + Name: "Topics", + Value: string(topics), + Selector: `request.payload.topics`, }, { - "name": "Timeout (ms)", - "value": fmt.Sprintf("%d", int(payload["TimeoutMs"].(float64))), + Name: "Timeout (ms)", + Value: fmt.Sprintf("%d", int(payload["timeoutMs"].(float64))), + Selector: `request.payload.timeoutMs`, }, { - "name": "Validate Only", - "value": validateOnly, + Name: "Validate Only", + Value: validateOnly, + Selector: `request.payload.validateOnly`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -579,26 +627,28 @@ func representCreateTopicsResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) - topics, _ := json.Marshal(payload["Topics"].([]interface{})) + payload := data["payload"].(map[string]interface{}) + topics, _ := json.Marshal(payload["topics"].([]interface{})) throttleTimeMs := "" - if payload["ThrottleTimeMs"] != nil { - throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64))) + if payload["throttleTimeMs"] != nil { + throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Throttle Time (ms)", - "value": throttleTimeMs, + Name: "Throttle Time (ms)", + Value: throttleTimeMs, + Selector: `response.payload.throttleTimeMs`, }, { - "name": "Topics", - "value": string(topics), + Name: "Topics", + Value: string(topics), + Selector: `response.payload.topics`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -609,35 +659,38 @@ func representDeleteTopicsRequest(data map[string]interface{}) []interface{} { rep = representRequestHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) + payload := data["payload"].(map[string]interface{}) topics := "" - if payload["Topics"] != nil { - x, _ := json.Marshal(payload["Topics"].([]interface{})) + if payload["topics"] != nil { + x, _ := json.Marshal(payload["topics"].([]interface{})) topics = string(x) } topicNames := "" - if payload["TopicNames"] != nil { - x, _ := json.Marshal(payload["TopicNames"].([]interface{})) + if payload["topicNames"] != nil { + x, _ := json.Marshal(payload["topicNames"].([]interface{})) topicNames = string(x) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "TopicNames", - "value": string(topicNames), + Name: "TopicNames", + Value: string(topicNames), + Selector: `request.payload.topicNames`, }, { - "name": "Topics", - "value": string(topics), + Name: "Topics", + Value: string(topics), + Selector: `request.payload.topics`, }, { - "name": "Timeout (ms)", - "value": fmt.Sprintf("%d", int(payload["TimeoutMs"].(float64))), + Name: "Timeout (ms)", + Value: fmt.Sprintf("%d", int(payload["timeoutMs"].(float64))), + Selector: `request.payload.timeoutMs`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep @@ -648,26 +701,28 @@ func representDeleteTopicsResponse(data map[string]interface{}) []interface{} { rep = representResponseHeader(data, rep) - payload := data["Payload"].(map[string]interface{}) - responses, _ := json.Marshal(payload["Responses"].([]interface{})) + payload := data["payload"].(map[string]interface{}) + responses, _ := json.Marshal(payload["responses"].([]interface{})) throttleTimeMs := "" - if payload["ThrottleTimeMs"] != nil { - throttleTimeMs = fmt.Sprintf("%d", int(payload["ThrottleTimeMs"].(float64))) + if payload["throttleTimeMs"] != nil { + throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64))) } - repPayload, _ := json.Marshal([]map[string]string{ + repPayload, _ := json.Marshal([]api.TableData{ { - "name": "Throttle Time (ms)", - "value": throttleTimeMs, + Name: "Throttle Time (ms)", + Value: throttleTimeMs, + Selector: `response.payload.throttleTimeMs`, }, { - "name": "Responses", - "value": string(responses), + Name: "Responses", + Value: string(responses), + Selector: `response.payload.responses`, }, }) - rep = append(rep, map[string]string{ - "type": api.TABLE, - "title": "Payload", - "data": string(repPayload), + rep = append(rep, api.SectionData{ + Type: api.TABLE, + Title: "Payload", + Data: string(repPayload), }) return rep diff --git a/tap/extensions/kafka/main.go b/tap/extensions/kafka/main.go index b677182d1..4c8ea0697 100644 --- a/tap/extensions/kafka/main.go +++ b/tap/extensions/kafka/main.go @@ -15,6 +15,7 @@ var _protocol api.Protocol = api.Protocol{ Name: "kafka", LongName: "Apache Kafka Protocol", Abbreviation: "KAFKA", + Macro: "kafka", Version: "12", BackgroundColor: "#000000", ForegroundColor: "#ffffff", @@ -61,7 +62,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co } } -func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry { +func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry { request := item.Pair.Request.Payload.(map[string]interface{}) reqDetails := request["details"].(map[string]interface{}) service := "kafka" @@ -70,76 +71,76 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve } else if resolvedSource != "" { service = resolvedSource } - apiKey := ApiKey(reqDetails["ApiKey"].(float64)) + apiKey := ApiKey(reqDetails["apiKey"].(float64)) summary := "" switch apiKey { case Metadata: - _topics := reqDetails["Payload"].(map[string]interface{})["Topics"] + _topics := reqDetails["payload"].(map[string]interface{})["topics"] if _topics == nil { break } topics := _topics.([]interface{}) for _, topic := range topics { - summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Name"].(string)) + summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["name"].(string)) } if len(summary) > 0 { summary = summary[:len(summary)-2] } break case ApiVersions: - summary = reqDetails["ClientID"].(string) + summary = reqDetails["clientID"].(string) break case Produce: - _topics := reqDetails["Payload"].(map[string]interface{})["TopicData"] + _topics := reqDetails["payload"].(map[string]interface{})["topicData"] if _topics == nil { break } topics := _topics.([]interface{}) for _, topic := range topics { - summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Topic"].(string)) + summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["topic"].(string)) } if len(summary) > 0 { summary = summary[:len(summary)-2] } break case Fetch: - _topics := reqDetails["Payload"].(map[string]interface{})["Topics"] + _topics := reqDetails["payload"].(map[string]interface{})["topics"] if _topics == nil { break } topics := _topics.([]interface{}) for _, topic := range topics { - summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Topic"].(string)) + summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["topic"].(string)) } if len(summary) > 0 { summary = summary[:len(summary)-2] } break case ListOffsets: - _topics := reqDetails["Payload"].(map[string]interface{})["Topics"] + _topics := reqDetails["payload"].(map[string]interface{})["topics"] if _topics == nil { break } topics := _topics.([]interface{}) for _, topic := range topics { - summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Name"].(string)) + summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["name"].(string)) } if len(summary) > 0 { summary = summary[:len(summary)-2] } break case CreateTopics: - topics := reqDetails["Payload"].(map[string]interface{})["Topics"].([]interface{}) + topics := reqDetails["payload"].(map[string]interface{})["topics"].([]interface{}) for _, topic := range topics { - summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["Name"].(string)) + summary += fmt.Sprintf("%s, ", topic.(map[string]interface{})["name"].(string)) } if len(summary) > 0 { summary = summary[:len(summary)-2] } break case DeleteTopics: - topicNames := reqDetails["TopicNames"].([]string) + topicNames := reqDetails["topicNames"].([]string) for _, name := range topicNames { summary += fmt.Sprintf("%s, ", name) } @@ -148,44 +149,48 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve request["url"] = summary elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds() - entryBytes, _ := json.Marshal(item.Pair) return &api.MizuEntry{ - ProtocolName: _protocol.Name, - ProtocolLongName: _protocol.LongName, - ProtocolAbbreviation: _protocol.Abbreviation, - ProtocolVersion: _protocol.Version, - ProtocolBackgroundColor: _protocol.BackgroundColor, - ProtocolForegroundColor: _protocol.ForegroundColor, - ProtocolFontSize: _protocol.FontSize, - ProtocolReferenceLink: _protocol.ReferenceLink, - EntryId: entryId, - Entry: string(entryBytes), - Url: fmt.Sprintf("%s%s", service, summary), - Method: apiNames[apiKey], - Status: 0, - RequestSenderIp: item.ConnectionInfo.ClientIP, - Service: service, - Timestamp: item.Timestamp, - ElapsedTime: elapsedTime, - Path: summary, - ResolvedSource: resolvedSource, - ResolvedDestination: resolvedDestination, - SourceIp: item.ConnectionInfo.ClientIP, - DestinationIp: item.ConnectionInfo.ServerIP, - SourcePort: item.ConnectionInfo.ClientPort, - DestinationPort: item.ConnectionInfo.ServerPort, - IsOutgoing: item.ConnectionInfo.IsOutgoing, + Protocol: _protocol, + Source: &api.TCP{ + Name: resolvedSource, + IP: item.ConnectionInfo.ClientIP, + Port: item.ConnectionInfo.ClientPort, + }, + Destination: &api.TCP{ + Name: resolvedDestination, + IP: item.ConnectionInfo.ServerIP, + Port: item.ConnectionInfo.ServerPort, + }, + Outgoing: item.ConnectionInfo.IsOutgoing, + Request: reqDetails, + Response: item.Pair.Response.Payload.(map[string]interface{})["details"].(map[string]interface{}), + Url: fmt.Sprintf("%s%s", service, summary), + Method: apiNames[apiKey], + Status: 0, + RequestSenderIp: item.ConnectionInfo.ClientIP, + Service: service, + Timestamp: item.Timestamp, + StartTime: item.Pair.Request.CaptureTime, + ElapsedTime: elapsedTime, + Summary: summary, + ResolvedSource: resolvedSource, + ResolvedDestination: resolvedDestination, + SourceIp: item.ConnectionInfo.ClientIP, + DestinationIp: item.ConnectionInfo.ServerIP, + SourcePort: item.ConnectionInfo.ClientPort, + DestinationPort: item.ConnectionInfo.ServerPort, + IsOutgoing: item.ConnectionInfo.IsOutgoing, } } func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { return &api.BaseEntryDetails{ - Id: entry.EntryId, + Id: entry.Id, Protocol: _protocol, Url: entry.Url, RequestSenderIp: entry.RequestSenderIp, Service: entry.Service, - Summary: entry.Path, + Summary: entry.Summary, StatusCode: entry.Status, Method: entry.Method, Timestamp: entry.Timestamp, @@ -202,49 +207,43 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { } } -func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) { - p = _protocol +func (d dissecting) Represent(protoIn api.Protocol, request map[string]interface{}, response map[string]interface{}) (protoOut api.Protocol, object []byte, bodySize int64, err error) { + protoOut = _protocol bodySize = 0 - var root map[string]interface{} - json.Unmarshal([]byte(entry.Entry), &root) representation := make(map[string]interface{}, 0) - request := root["request"].(map[string]interface{})["payload"].(map[string]interface{}) - response := root["response"].(map[string]interface{})["payload"].(map[string]interface{}) - reqDetails := request["details"].(map[string]interface{}) - resDetails := response["details"].(map[string]interface{}) - apiKey := ApiKey(reqDetails["ApiKey"].(float64)) + apiKey := ApiKey(request["apiKey"].(float64)) var repRequest []interface{} var repResponse []interface{} switch apiKey { case Metadata: - repRequest = representMetadataRequest(reqDetails) - repResponse = representMetadataResponse(resDetails) + repRequest = representMetadataRequest(request) + repResponse = representMetadataResponse(response) break case ApiVersions: - repRequest = representApiVersionsRequest(reqDetails) - repResponse = representApiVersionsResponse(resDetails) + repRequest = representApiVersionsRequest(request) + repResponse = representApiVersionsResponse(response) break case Produce: - repRequest = representProduceRequest(reqDetails) - repResponse = representProduceResponse(resDetails) + repRequest = representProduceRequest(request) + repResponse = representProduceResponse(response) break case Fetch: - repRequest = representFetchRequest(reqDetails) - repResponse = representFetchResponse(resDetails) + repRequest = representFetchRequest(request) + repResponse = representFetchResponse(response) break case ListOffsets: - repRequest = representListOffsetsRequest(reqDetails) - repResponse = representListOffsetsResponse(resDetails) + repRequest = representListOffsetsRequest(request) + repResponse = representListOffsetsResponse(response) break case CreateTopics: - repRequest = representCreateTopicsRequest(reqDetails) - repResponse = representCreateTopicsResponse(resDetails) + repRequest = representCreateTopicsRequest(request) + repResponse = representCreateTopicsResponse(response) break case DeleteTopics: - repRequest = representDeleteTopicsRequest(reqDetails) - repResponse = representDeleteTopicsResponse(resDetails) + repRequest = representDeleteTopicsRequest(request) + repResponse = representDeleteTopicsResponse(response) break } @@ -254,4 +253,10 @@ func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []by return } +func (d dissecting) Macros() map[string]string { + return map[string]string{ + `kafka`: fmt.Sprintf(`proto.abbr == "%s"`, _protocol.Abbreviation), + } +} + var Dissector dissecting diff --git a/tap/extensions/kafka/request.go b/tap/extensions/kafka/request.go index b7ac67d7d..263755892 100644 --- a/tap/extensions/kafka/request.go +++ b/tap/extensions/kafka/request.go @@ -10,13 +10,13 @@ import ( ) type Request struct { - Size int32 - ApiKey ApiKey - ApiVersion int16 - CorrelationID int32 - ClientID string - Payload interface{} - CaptureTime time.Time + Size int32 `json:"size"` + ApiKey ApiKey `json:"apiKey"` + ApiVersion int16 `json:"apiVersion"` + CorrelationID int32 `json:"correlationID"` + ClientID string `json:"clientID"` + Payload interface{} `json:"payload"` + CaptureTime time.Time `json:"captureTime"` } func ReadRequest(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer) (apiKey ApiKey, apiVersion int16, err error) { diff --git a/tap/extensions/kafka/response.go b/tap/extensions/kafka/response.go index 574efa8a2..5441e784d 100644 --- a/tap/extensions/kafka/response.go +++ b/tap/extensions/kafka/response.go @@ -10,10 +10,10 @@ import ( ) type Response struct { - Size int32 - CorrelationID int32 - Payload interface{} - CaptureTime time.Time + Size int32 `json:"size"` + CorrelationID int32 `json:"correlationID"` + Payload interface{} `json:"payload"` + CaptureTime time.Time `json:"captureTime"` } func ReadResponse(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter) (err error) { diff --git a/tap/extensions/kafka/structs.go b/tap/extensions/kafka/structs.go index d9aa5c1cb..67064d85e 100644 --- a/tap/extensions/kafka/structs.go +++ b/tap/extensions/kafka/structs.go @@ -26,38 +26,38 @@ func (acks RequiredAcks) String() string { } type UUID struct { - TimeLow int32 - TimeMid int16 - TimeHiAndVersion int16 - ClockSeq int16 - NodePart1 int32 - NodePart22 int16 + TimeLow int32 `json:"timeLow"` + TimeMid int16 `json:"timeMid"` + TimeHiAndVersion int16 `json:"timeHiAndVersion"` + ClockSeq int16 `json:"clockSeq"` + NodePart1 int32 `json:"nodePart1"` + NodePart22 int16 `json:"nodePart22"` } // Metadata Request (Version: 0) type MetadataRequestTopicV0 struct { - Name string + Name string `json:"name"` } type MetadataRequestV0 struct { - Topics []MetadataRequestTopicV0 + Topics []MetadataRequestTopicV0 `json:"topics"` } // Metadata Request (Version: 4) type MetadataRequestV4 struct { - Topics []MetadataRequestTopicV0 - AllowAutoTopicCreation bool + Topics []MetadataRequestTopicV0 `json:"topics"` + AllowAutoTopicCreation bool `json:"allowAutoTopicCreation"` } // Metadata Request (Version: 8) type MetadataRequestV8 struct { - Topics []MetadataRequestTopicV0 - AllowAutoTopicCreation bool - IncludeClusterAuthorizedOperations bool - IncludeTopicAuthorizedOperations bool + Topics []MetadataRequestTopicV0 `json:"topics"` + AllowAutoTopicCreation bool `json:"allowAutoTopicCreation"` + IncludeClusterAuthorizedOperations bool `json:"includeClusterAuthorizedOperations"` + IncludeTopicAuthorizedOperations bool `json:"includeTopicAuthorizedOperations"` } // Metadata Request (Version: 10) @@ -68,188 +68,188 @@ type MetadataRequestTopicV10 struct { } type MetadataRequestV10 struct { - Topics []MetadataRequestTopicV10 - AllowAutoTopicCreation bool - IncludeClusterAuthorizedOperations bool - IncludeTopicAuthorizedOperations bool + Topics []MetadataRequestTopicV10 `json:"topics"` + AllowAutoTopicCreation bool `json:"allowAutoTopicCreation"` + IncludeClusterAuthorizedOperations bool `json:"includeClusterAuthorizedOperations"` + IncludeTopicAuthorizedOperations bool `json:"includeTopicAuthorizedOperations"` } // Metadata Request (Version: 11) type MetadataRequestV11 struct { - Topics []MetadataRequestTopicV10 - AllowAutoTopicCreation bool - IncludeTopicAuthorizedOperations bool + Topics []MetadataRequestTopicV10 `json:"topics"` + AllowAutoTopicCreation bool `json:"allowAutoTopicCreation"` + IncludeTopicAuthorizedOperations bool `json:"includeTopicAuthorizedOperations"` } // Metadata Response (Version: 0) type BrokerV0 struct { - NodeId int32 - Host string - Port int32 + NodeId int32 `json:"nodeId"` + Host string `json:"host"` + Port int32 `json:"port"` } type PartitionsV0 struct { - ErrorCode int16 - PartitionIndex int32 - LeaderId int32 - ReplicaNodes int32 - IsrNodes int32 + ErrorCode int16 `json:"errorCode"` + PartitionIndex int32 `json:"partitionIndex"` + LeaderId int32 `json:"leaderId"` + ReplicaNodes int32 `json:"replicaNodes"` + IsrNodes int32 `json:"isrNodes"` } type TopicV0 struct { - ErrorCode int16 - Name string - Partitions []PartitionsV0 + ErrorCode int16 `json:"errorCode"` + Name string `json:"name"` + Partitions []PartitionsV0 `json:"partitions"` } type MetadataResponseV0 struct { - Brokers []BrokerV0 - Topics []TopicV0 + Brokers []BrokerV0 `json:"brokers"` + Topics []TopicV0 `json:"topics"` } // Metadata Response (Version: 1) type BrokerV1 struct { - NodeId int32 - Host string - Port int32 - Rack string + NodeId int32 `json:"nodeId"` + Host string `json:"host"` + Port int32 `json:"port"` + Rack string `json:"rack"` } type TopicV1 struct { - ErrorCode int16 - Name string - IsInternal bool - Partitions []PartitionsV0 + ErrorCode int16 `json:"errorCode"` + Name string `json:"name"` + IsInternal bool `json:"isInternal"` + Partitions []PartitionsV0 `json:"partitions"` } type MetadataResponseV1 struct { - Brokers []BrokerV1 - ControllerID int32 - Topics []TopicV1 + Brokers []BrokerV1 `json:"brokers"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV1 `json:"topics"` } // Metadata Response (Version: 2) type MetadataResponseV2 struct { - Brokers []BrokerV1 - ClusterID string - ControllerID int32 - Topics []TopicV1 + Brokers []BrokerV1 `json:"brokers"` + ClusterID string `json:"clusterID"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV1 `json:"topics"` } // Metadata Response (Version: 3) type MetadataResponseV3 struct { - ThrottleTimeMs int32 - Brokers []BrokerV1 - ClusterID string - ControllerID int32 - Topics []TopicV1 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Brokers []BrokerV1 `json:"brokers"` + ClusterID string `json:"clusterID"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV1 `json:"topics"` } // Metadata Response (Version: 5) type PartitionsV5 struct { - ErrorCode int16 - PartitionIndex int32 - LeaderId int32 - ReplicaNodes int32 - IsrNodes int32 - OfflineReplicas int32 + ErrorCode int16 `json:"errorCode"` + PartitionIndex int32 `json:"partitionIndex"` + LeaderId int32 `json:"leaderId"` + ReplicaNodes int32 `json:"replicaNodes"` + IsrNodes int32 `json:"isrNodes"` + OfflineReplicas int32 `json:"offlineReplicas"` } type TopicV5 struct { - ErrorCode int16 - Name string - IsInternal bool - Partitions []PartitionsV5 + ErrorCode int16 `json:"errorCode"` + Name string `json:"name"` + IsInternal bool `json:"isInternal"` + Partitions []PartitionsV5 `json:"partitions"` } type MetadataResponseV5 struct { - ThrottleTimeMs int32 - Brokers []BrokerV1 - ClusterID string - ControllerID int32 - Topics []TopicV5 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Brokers []BrokerV1 `json:"brokers"` + ClusterID string `json:"clusterID"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV5 `json:"topics"` } // Metadata Response (Version: 7) type PartitionsV7 struct { - ErrorCode int16 - PartitionIndex int32 - LeaderId int32 - LeaderEpoch int32 - ReplicaNodes int32 - IsrNodes int32 - OfflineReplicas int32 + ErrorCode int16 `json:"errorCode"` + PartitionIndex int32 `json:"partitionIndex"` + LeaderId int32 `json:"leaderId"` + LeaderEpoch int32 `json:"leaderEpoch"` + ReplicaNodes int32 `json:"replicaNodes"` + IsrNodes int32 `json:"isrNodes"` + OfflineReplicas int32 `json:"offlineReplicas"` } type TopicV7 struct { - ErrorCode int16 - Name string - IsInternal bool - Partitions []PartitionsV7 + ErrorCode int16 `json:"errorCode"` + Name string `json:"name"` + IsInternal bool `json:"isInternal"` + Partitions []PartitionsV7 `json:"partitions"` } type MetadataResponseV7 struct { - ThrottleTimeMs int32 - Brokers []BrokerV1 - ClusterID string - ControllerID int32 - Topics []TopicV7 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Brokers []BrokerV1 `json:"brokers"` + ClusterID string `json:"clusterID"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV7 `json:"topics"` } // Metadata Response (Version: 8) type TopicV8 struct { - ErrorCode int16 - Name string - IsInternal bool - Partitions []PartitionsV7 - TopicAuthorizedOperations int32 + ErrorCode int16 `json:"errorCode"` + Name string `json:"name"` + IsInternal bool `json:"isInternal"` + Partitions []PartitionsV7 `json:"partitions"` + TopicAuthorizedOperations int32 `json:"topicAuthorizedOperations"` } type MetadataResponseV8 struct { - ThrottleTimeMs int32 - Brokers []BrokerV1 - ClusterID string - ControllerID int32 - Topics []TopicV8 - ClusterAuthorizedOperations int32 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Brokers []BrokerV1 `json:"brokers"` + ClusterID string `json:"clusterID"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV8 `json:"topics"` + ClusterAuthorizedOperations int32 `json:"clusterAuthorizedOperations"` } // Metadata Response (Version: 10) type TopicV10 struct { - ErrorCode int16 - Name string - TopicID UUID - IsInternal bool - Partitions []PartitionsV7 - TopicAuthorizedOperations int32 + ErrorCode int16 `json:"errorCode"` + Name string `json:"name"` + TopicID UUID `json:"topicID"` + IsInternal bool `json:"isInternal"` + Partitions []PartitionsV7 `json:"partitions"` + TopicAuthorizedOperations int32 `json:"topicAuthorizedOperations"` } type MetadataResponseV10 struct { - ThrottleTimeMs int32 - Brokers []BrokerV1 - ClusterID string - ControllerID int32 - Topics []TopicV10 - ClusterAuthorizedOperations int32 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Brokers []BrokerV1 `json:"brokers"` + ClusterID string `json:"clusterID"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV10 `json:"topics"` + ClusterAuthorizedOperations int32 `json:"clusterAuthorizedOperations"` } // Metadata Response (Version: 11) type MetadataResponseV11 struct { - ThrottleTimeMs int32 - Brokers []BrokerV1 - ClusterID string - ControllerID int32 - Topics []TopicV10 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Brokers []BrokerV1 `json:"brokers"` + ClusterID string `json:"clusterID"` + ControllerID int32 `json:"controllerID"` + Topics []TopicV10 `json:"topics"` } // ApiVersions Request (Version: 0) @@ -259,742 +259,742 @@ type ApiVersionsRequestV0 struct{} // ApiVersions Request (Version: 3) type ApiVersionsRequestV3 struct { - ClientSoftwareName string - ClientSoftwareVersion string + ClientSoftwareName string `json:"clientSoftwareName"` + ClientSoftwareVersion string `json:"clientSoftwareVersion"` } // ApiVersions Response (Version: 0) type ApiVersionsResponseApiKey struct { - ApiKey int16 - MinVersion int16 - MaxVersion int16 + ApiKey int16 `json:"apiKey"` + MinVersion int16 `json:"minVersion"` + MaxVersion int16 `json:"maxVersion"` } type ApiVersionsResponseV0 struct { - ErrorCode int16 - ApiKeys []ApiVersionsResponseApiKey + ErrorCode int16 `json:"errorCode"` + ApiKeys []ApiVersionsResponseApiKey `json:"apiKeys"` } // ApiVersions Response (Version: 1) type ApiVersionsResponseV1 struct { - ErrorCode int16 - ApiKeys []ApiVersionsResponseApiKey // FIXME: `confluent-kafka-python` causes memory leak - ThrottleTimeMs int32 + ErrorCode int16 `json:"errorCode"` + ApiKeys []ApiVersionsResponseApiKey `json:"apiKeys"` // FIXME: `confluent-kafka-python` causes memory leak + ThrottleTimeMs int32 `json:"throttleTimeMs"` } // Produce Request (Version: 0) // Message is a kafka message type type MessageV0 struct { - Codec int8 // codec used to compress the message contents - CompressionLevel int // compression level - LogAppendTime bool // the used timestamp is LogAppendTime - Key []byte // the message key, may be nil - Value []byte // the message contents - Set *MessageSet // the message set a message might wrap - Version int8 // v1 requires Kafka 0.10 - Timestamp time.Time // the timestamp of the message (version 1+ only) + Codec int8 `json:"codec"` // codec used to compress the message contents + CompressionLevel int `json:"compressionLevel"` // compression level + LogAppendTime bool `json:"logAppendTime"` // the used timestamp is LogAppendTime + Key []byte `json:"key"` // the message key, may be nil + Value []byte `json:"value"` // the message contents + Set *MessageSet `json:"set"` // the message set a message might wrap + Version int8 `json:"version"` // v1 requires Kafka 0.10 + Timestamp time.Time `json:"timestamp"` // the timestamp of the message (version 1+ only) compressedSize int // used for computing the compression ratio metrics } // MessageBlock represents a part of request with message type MessageBlock struct { - Offset int64 - Msg *MessageV0 + Offset int64 `json:"offset"` + Msg *MessageV0 `json:"msg"` } // MessageSet is a replacement for RecordBatch in older versions type MessageSet struct { - PartialTrailingMessage bool // whether the set on the wire contained an incomplete trailing MessageBlock - OverflowMessage bool // whether the set on the wire contained an overflow message - Messages []*MessageBlock + PartialTrailingMessage bool `json:"partialTrailingMessage"` // whether the set on the wire contained an incomplete trailing MessageBlock + OverflowMessage bool `json:"overflowMessage"` // whether the set on the wire contained an overflow message + Messages []*MessageBlock `json:"messages"` } type RecordHeader struct { - HeaderKeyLength int8 - HeaderKey string - HeaderValueLength int8 - Value string + HeaderKeyLength int8 `json:"headerKeyLength"` + HeaderKey string `json:"headerKey"` + HeaderValueLength int8 `json:"headerValueLength"` + Value string `json:"value"` } // Record is kafka record type type RecordV0 struct { - Unknown int8 - Attributes int8 - TimestampDelta int8 - OffsetDelta int8 - KeyLength int8 - Key string - ValueLen int8 - Value string - Headers []RecordHeader + Unknown int8 `json:"unknown"` + Attributes int8 `json:"attributes"` + TimestampDelta int8 `json:"timestampDelta"` + OffsetDelta int8 `json:"offsetDelta"` + KeyLength int8 `json:"keyLength"` + Key string `json:"key"` + ValueLen int8 `json:"valueLen"` + Value string `json:"value"` + Headers []RecordHeader `json:"headers"` } // RecordBatch are records from one kafka request type RecordBatch struct { - BaseOffset int64 - BatchLength int32 - PartitionLeaderEpoch int32 - Magic int8 - Crc int32 - Attributes int16 - LastOffsetDelta int32 - FirstTimestamp int64 - MaxTimestamp int64 - ProducerId int64 - ProducerEpoch int16 - BaseSequence int32 - Record []RecordV0 + BaseOffset int64 `json:"baseOffset"` + BatchLength int32 `json:"batchLength"` + PartitionLeaderEpoch int32 `json:"partitionLeaderEpoch"` + Magic int8 `json:"magic"` + Crc int32 `json:"crc"` + Attributes int16 `json:"attributes"` + LastOffsetDelta int32 `json:"lastOffsetDelta"` + FirstTimestamp int64 `json:"firstTimestamp"` + MaxTimestamp int64 `json:"maxTimestamp"` + ProducerId int64 `json:"producerId"` + ProducerEpoch int16 `json:"producerEpoch"` + BaseSequence int32 `json:"baseSequence"` + Record []RecordV0 `json:"record"` } type Records struct { - RecordBatch RecordBatch + RecordBatch RecordBatch `json:"recordBatch"` // TODO: Implement `MessageSet` // MessageSet MessageSet } type PartitionData struct { - Index int32 - Unknown int32 - Records Records + Index int32 `json:"index"` + Unknown int32 `json:"unknown"` + Records Records `json:"records"` } type Partitions struct { - Length int32 - PartitionData PartitionData + Length int32 `json:"length"` + PartitionData PartitionData `json:"partitionData"` } type TopicData struct { - Topic string - Partitions Partitions + Topic string `json:"topic"` + Partitions Partitions `json:"partitions"` } type ProduceRequestV0 struct { - RequiredAcks RequiredAcks - Timeout int32 - TopicData []TopicData + RequiredAcks RequiredAcks `json:"requiredAcks"` + Timeout int32 `json:"timeout"` + TopicData []TopicData `json:"topicData"` } // Produce Request (Version: 3) type ProduceRequestV3 struct { - TransactionalID string - RequiredAcks RequiredAcks - Timeout int32 - TopicData []TopicData + TransactionalID string `json:"transactionalID"` + RequiredAcks RequiredAcks `json:"requiredAcks"` + Timeout int32 `json:"timeout"` + TopicData []TopicData `json:"topicData"` } // Produce Response (Version: 0) type PartitionResponseV0 struct { - Index int32 - ErrorCode int16 - BaseOffset int64 + Index int32 `json:"index"` + ErrorCode int16 `json:"errorCode"` + BaseOffset int64 `json:"baseOffset"` } type ResponseV0 struct { - Name string - PartitionResponses []PartitionResponseV0 + Name string `json:"name"` + PartitionResponses []PartitionResponseV0 `json:"partitionResponses"` } type ProduceResponseV0 struct { - Responses []ResponseV0 + Responses []ResponseV0 `json:"responses"` } // Produce Response (Version: 1) type ProduceResponseV1 struct { - Responses []ResponseV0 - ThrottleTimeMs int32 + Responses []ResponseV0 `json:"responses"` + ThrottleTimeMs int32 `json:"throttleTimeMs"` } // Produce Response (Version: 2) type PartitionResponseV2 struct { - Index int32 - ErrorCode int16 - BaseOffset int64 - LogAppendTimeMs int64 + Index int32 `json:"index"` + ErrorCode int16 `json:"errorCode"` + BaseOffset int64 `json:"baseOffset"` + LogAppendTimeMs int64 `json:"logAppendTimeMs"` } type ResponseV2 struct { - Name string - PartitionResponses []PartitionResponseV2 + Name string `json:"name"` + PartitionResponses []PartitionResponseV2 `json:"partitionResponses"` } type ProduceResponseV2 struct { - Responses []ResponseV2 - ThrottleTimeMs int32 + Responses []ResponseV2 `json:"responses"` + ThrottleTimeMs int32 `json:"throttleTimeMs"` } // Produce Response (Version: 5) type PartitionResponseV5 struct { - Index int32 - ErrorCode int16 - BaseOffset int64 - LogAppendTimeMs int64 - LogStartOffset int64 + Index int32 `json:"index"` + ErrorCode int16 `json:"errorCode"` + BaseOffset int64 `json:"baseOffset"` + LogAppendTimeMs int64 `json:"logAppendTimeMs"` + LogStartOffset int64 `json:"logStartOffset"` } type ResponseV5 struct { - Name string - PartitionResponses []PartitionResponseV5 + Name string `json:"name"` + PartitionResponses []PartitionResponseV5 `json:"partitionResponses"` } type ProduceResponseV5 struct { - Responses []ResponseV5 - ThrottleTimeMs int32 + Responses []ResponseV5 `json:"responses"` + ThrottleTimeMs int32 `json:"throttleTimeMs"` } // Produce Response (Version: 8) type RecordErrors struct { - BatchIndex int32 - BatchIndexErrorMessage string + BatchIndex int32 `json:"batchIndex"` + BatchIndexErrorMessage string `json:"batchIndexErrorMessage"` } type PartitionResponseV8 struct { - Index int32 - ErrorCode int16 - BaseOffset int64 - LogAppendTimeMs int64 - LogStartOffset int64 - RecordErrors RecordErrors - ErrorMessage string + Index int32 `json:"index"` + ErrorCode int16 `json:"errorCode"` + BaseOffset int64 `json:"baseOffset"` + LogAppendTimeMs int64 `json:"logAppendTimeMs"` + LogStartOffset int64 `json:"logStartOffset"` + RecordErrors RecordErrors `json:"recordErrors"` + ErrorMessage string `json:"errorMessage"` } type ResponseV8 struct { - Name string - PartitionResponses []PartitionResponseV8 + Name string `json:"name"` + PartitionResponses []PartitionResponseV8 `json:"partitionResponses"` } type ProduceResponseV8 struct { - Responses []ResponseV8 - ThrottleTimeMs int32 + Responses []ResponseV8 `json:"responses"` + ThrottleTimeMs int32 `json:"throttleTimeMs"` } // Fetch Request (Version: 0) type FetchPartitionV0 struct { - Partition int32 - FetchOffset int64 - PartitionMaxBytes int32 + Partition int32 `json:"partition"` + FetchOffset int64 `json:"fetchOffset"` + PartitionMaxBytes int32 `json:"partitionMaxBytes"` } type FetchTopicV0 struct { - Topic string - Partitions []FetchPartitionV0 + Topic string `json:"topic"` + Partitions []FetchPartitionV0 `json:"partitions"` } type FetchRequestV0 struct { - ReplicaId int32 - MaxWaitMs int32 - MinBytes int32 - Topics []FetchTopicV0 + ReplicaId int32 `json:"replicaId"` + MaxWaitMs int32 `json:"maxWaitMs"` + MinBytes int32 `json:"minBytes"` + Topics []FetchTopicV0 `json:"topics"` } // Fetch Request (Version: 3) type FetchRequestV3 struct { - ReplicaId int32 - MaxWaitMs int32 - MinBytes int32 - MaxBytes int32 - Topics []FetchTopicV0 + ReplicaId int32 `json:"replicaId"` + MaxWaitMs int32 `json:"maxWaitMs"` + MinBytes int32 `json:"minBytes"` + MaxBytes int32 `json:"maxBytes"` + Topics []FetchTopicV0 `json:"topics"` } // Fetch Request (Version: 4) type FetchRequestV4 struct { - ReplicaId int32 - MaxWaitMs int32 - MinBytes int32 - MaxBytes int32 - IsolationLevel int8 - Topics []FetchTopicV0 + ReplicaId int32 `json:"replicaId"` + MaxWaitMs int32 `json:"maxWaitMs"` + MinBytes int32 `json:"minBytes"` + MaxBytes int32 `json:"maxBytes"` + IsolationLevel int8 `json:"isolationLevel"` + Topics []FetchTopicV0 `json:"topics"` } // Fetch Request (Version: 5) type FetchPartitionV5 struct { - Partition int32 - FetchOffset int64 - LogStartOffset int64 - PartitionMaxBytes int32 + Partition int32 `json:"partition"` + FetchOffset int64 `json:"fetchOffset"` + LogStartOffset int64 `json:"logStartOffset"` + PartitionMaxBytes int32 `json:"partitionMaxBytes"` } type FetchTopicV5 struct { - Topic string - Partitions []FetchPartitionV5 + Topic string `json:"topic"` + Partitions []FetchPartitionV5 `json:"partitions"` } type FetchRequestV5 struct { - ReplicaId int32 - MaxWaitMs int32 - MinBytes int32 - MaxBytes int32 - IsolationLevel int8 - Topics []FetchTopicV5 + ReplicaId int32 `json:"replicaId"` + MaxWaitMs int32 `json:"maxWaitMs"` + MinBytes int32 `json:"minBytes"` + MaxBytes int32 `json:"maxBytes"` + IsolationLevel int8 `json:"isolationLevel"` + Topics []FetchTopicV5 `json:"topics"` } // Fetch Request (Version: 7) type ForgottenTopicsDataV7 struct { - Topic string - Partitions []int32 + Topic string `json:"topic"` + Partitions []int32 `json:"partitions"` } type FetchRequestV7 struct { - ReplicaId int32 - MaxWaitMs int32 - MinBytes int32 - MaxBytes int32 - IsolationLevel int8 - SessionId int32 - SessionEpoch int32 - Topics []FetchTopicV5 - ForgottenTopicsData ForgottenTopicsDataV7 + ReplicaId int32 `json:"replicaId"` + MaxWaitMs int32 `json:"maxWaitMs"` + MinBytes int32 `json:"minBytes"` + MaxBytes int32 `json:"maxBytes"` + IsolationLevel int8 `json:"isolationLevel"` + SessionId int32 `json:"sessionId"` + SessionEpoch int32 `json:"sessionEpoch"` + Topics []FetchTopicV5 `json:"topics"` + ForgottenTopicsData ForgottenTopicsDataV7 `json:"forgottenTopicsData"` } // Fetch Request (Version: 9) type FetchPartitionV9 struct { - Partition int32 - CurrentLeaderEpoch int32 - FetchOffset int64 - LogStartOffset int64 - PartitionMaxBytes int32 + Partition int32 `json:"partition"` + CurrentLeaderEpoch int32 `json:"currentLeaderEpoch"` + FetchOffset int64 `json:"fetchOffset"` + LogStartOffset int64 `json:"logStartOffset"` + PartitionMaxBytes int32 `json:"partitionMaxBytes"` } type FetchTopicV9 struct { - Topic string - Partitions []FetchPartitionV9 + Topic string `json:"topic"` + Partitions []FetchPartitionV9 `json:"partitions"` } type FetchRequestV9 struct { - ReplicaId int32 - MaxWaitMs int32 - MinBytes int32 - MaxBytes int32 - IsolationLevel int8 - SessionId int32 - SessionEpoch int32 - Topics []FetchTopicV9 - ForgottenTopicsData ForgottenTopicsDataV7 + ReplicaId int32 `json:"replicaId"` + MaxWaitMs int32 `json:"maxWaitMs"` + MinBytes int32 `json:"minBytes"` + MaxBytes int32 `json:"maxBytes"` + IsolationLevel int8 `json:"isolationLevel"` + SessionId int32 `json:"sessionId"` + SessionEpoch int32 `json:"sessionEpoch"` + Topics []FetchTopicV9 `json:"topics"` + ForgottenTopicsData ForgottenTopicsDataV7 `json:"forgottenTopicsData"` } // Fetch Request (Version: 11) type FetchRequestV11 struct { - ReplicaId int32 - MaxWaitMs int32 - MinBytes int32 - MaxBytes int32 - IsolationLevel int8 - SessionId int32 - SessionEpoch int32 - Topics []FetchTopicV9 - ForgottenTopicsData ForgottenTopicsDataV7 - RackId string + ReplicaId int32 `json:"replicaId"` + MaxWaitMs int32 `json:"maxWaitMs"` + MinBytes int32 `json:"minBytes"` + MaxBytes int32 `json:"maxBytes"` + IsolationLevel int8 `json:"isolationLevel"` + SessionId int32 `json:"sessionId"` + SessionEpoch int32 `json:"sessionEpoch"` + Topics []FetchTopicV9 `json:"topics"` + ForgottenTopicsData ForgottenTopicsDataV7 `json:"forgottenTopicsData"` + RackId string `json:"rackId"` } // Fetch Response (Version: 0) type PartitionResponseFetchV0 struct { - Partition int32 - ErrorCode int16 - HighWatermark int64 - RecordSet Records + Partition int32 `json:"partition"` + ErrorCode int16 `json:"errorCode"` + HighWatermark int64 `json:"highWatermark"` + RecordSet Records `json:"recordSet"` } type ResponseFetchV0 struct { - Topic string - PartitionResponses []PartitionResponseFetchV0 + Topic string `json:"topic"` + PartitionResponses []PartitionResponseFetchV0 `json:"partitionResponses"` } type FetchResponseV0 struct { - Responses []ResponseFetchV0 + Responses []ResponseFetchV0 `json:"responses"` } // Fetch Response (Version: 1) type FetchResponseV1 struct { - ThrottleTimeMs int32 - Responses []ResponseFetchV0 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Responses []ResponseFetchV0 `json:"responses"` } // Fetch Response (Version: 4) type AbortedTransactionsV4 struct { - ProducerId int32 - FirstOffset int32 + ProducerId int32 `json:"producerId"` + FirstOffset int32 `json:"firstOffset"` } type PartitionResponseFetchV4 struct { - Partition int32 - ErrorCode int16 - HighWatermark int64 - LastStableOffset int64 - AbortedTransactions AbortedTransactionsV4 - RecordSet Records + Partition int32 `json:"partition"` + ErrorCode int16 `json:"errorCode"` + HighWatermark int64 `json:"highWatermark"` + LastStableOffset int64 `json:"lastStableOffset"` + AbortedTransactions AbortedTransactionsV4 `json:"abortedTransactions"` + RecordSet Records `json:"recordSet"` } type ResponseFetchV4 struct { - Topic string - PartitionResponses []PartitionResponseFetchV4 + Topic string `json:"topic"` + PartitionResponses []PartitionResponseFetchV4 `json:"partitionResponses"` } type FetchResponseV4 struct { - ThrottleTimeMs int32 - Responses []ResponseFetchV4 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Responses []ResponseFetchV4 `json:"responses"` } // Fetch Response (Version: 5) type PartitionResponseFetchV5 struct { - Partition int32 - ErrorCode int16 - HighWatermark int64 - LastStableOffset int64 - LogStartOffset int64 - AbortedTransactions AbortedTransactionsV4 - RecordSet Records + Partition int32 `json:"partition"` + ErrorCode int16 `json:"errorCode"` + HighWatermark int64 `json:"highWatermark"` + LastStableOffset int64 `json:"lastStableOffset"` + LogStartOffset int64 `json:"logStartOffset"` + AbortedTransactions AbortedTransactionsV4 `json:"abortedTransactions"` + RecordSet Records `json:"recordSet"` } type ResponseFetchV5 struct { - Topic string - PartitionResponses []PartitionResponseFetchV5 + Topic string `json:"topic"` + PartitionResponses []PartitionResponseFetchV5 `json:"partitionResponses"` } type FetchResponseV5 struct { - ThrottleTimeMs int32 - Responses []ResponseFetchV5 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Responses []ResponseFetchV5 `json:"responses"` } // Fetch Response (Version: 7) type FetchResponseV7 struct { - ThrottleTimeMs int32 - ErrorCode int16 - SessionId int32 - Responses []ResponseFetchV5 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + ErrorCode int16 `json:"errorCode"` + SessionId int32 `json:"sessionId"` + Responses []ResponseFetchV5 `json:"responses"` } // Fetch Response (Version: 11) type PartitionResponseFetchV11 struct { - Partition int32 - ErrorCode int16 - HighWatermark int64 - LastStableOffset int64 - LogStartOffset int64 - AbortedTransactions AbortedTransactionsV4 - PreferredReadReplica int32 - RecordSet Records + Partition int32 `json:"partition"` + ErrorCode int16 `json:"errorCode"` + HighWatermark int64 `json:"highWatermark"` + LastStableOffset int64 `json:"lastStableOffset"` + LogStartOffset int64 `json:"logStartOffset"` + AbortedTransactions AbortedTransactionsV4 `json:"abortedTransactions"` + PreferredReadReplica int32 `json:"preferredReadReplica"` + RecordSet Records `json:"recordSet"` } type ResponseFetchV11 struct { - Topic string - PartitionResponses []PartitionResponseFetchV11 + Topic string `json:"topic"` + PartitionResponses []PartitionResponseFetchV11 `json:"partitionResponses"` } type FetchResponseV11 struct { - ThrottleTimeMs int32 - ErrorCode int16 - SessionId int32 - Responses []ResponseFetchV5 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + ErrorCode int16 `json:"errorCode"` + SessionId int32 `json:"sessionId"` + Responses []ResponseFetchV5 `json:"responses"` } // ListOffsets Request (Version: 0) type ListOffsetsRequestPartitionV0 struct { - PartitionIndex int32 - Timestamp int64 - MaxNumOffsets int32 + PartitionIndex int32 `json:"partitionIndex"` + Timestamp int64 `json:"timestamp"` + MaxNumOffsets int32 `json:"maxNumOffsets"` } type ListOffsetsRequestTopicV0 struct { - Name string - Partitions []ListOffsetsRequestPartitionV0 + Name string `json:"name"` + Partitions []ListOffsetsRequestPartitionV0 `json:"partitions"` } type ListOffsetsRequestV0 struct { - ReplicaId int32 - Topics []ListOffsetsRequestTopicV0 + ReplicaId int32 `json:"replicaId"` + Topics []ListOffsetsRequestTopicV0 `json:"topics"` } // ListOffsets Request (Version: 1) type ListOffsetsRequestPartitionV1 struct { - PartitionIndex int32 - Timestamp int64 + PartitionIndex int32 `json:"partitionIndex"` + Timestamp int64 `json:"timestamp"` } type ListOffsetsRequestTopicV1 struct { - Name string - Partitions []ListOffsetsRequestPartitionV1 + Name string `json:"name"` + Partitions []ListOffsetsRequestPartitionV1 `json:"partitions"` } type ListOffsetsRequestV1 struct { - ReplicaId int32 - Topics []ListOffsetsRequestTopicV1 + ReplicaId int32 `json:"replicaId"` + Topics []ListOffsetsRequestTopicV1 `json:"topics"` } // ListOffsets Request (Version: 2) type ListOffsetsRequestV2 struct { - ReplicaId int32 - IsolationLevel int8 - Topics []ListOffsetsRequestTopicV1 + ReplicaId int32 `json:"replicaId"` + IsolationLevel int8 `json:"isolationLevel"` + Topics []ListOffsetsRequestTopicV1 `json:"topics"` } // ListOffsets Request (Version: 4) type ListOffsetsRequestPartitionV4 struct { - PartitionIndex int32 - CurrentLeaderEpoch int32 - Timestamp int64 + PartitionIndex int32 `json:"partitionIndex"` + CurrentLeaderEpoch int32 `json:"currentLeaderEpoch"` + Timestamp int64 `json:"timestamp"` } type ListOffsetsRequestTopicV4 struct { - Name string - Partitions []ListOffsetsRequestPartitionV4 + Name string `json:"name"` + Partitions []ListOffsetsRequestPartitionV4 `json:"partitions"` } type ListOffsetsRequestV4 struct { - ReplicaId int32 - Topics []ListOffsetsRequestTopicV4 + ReplicaId int32 `json:"replicaId"` + Topics []ListOffsetsRequestTopicV4 `json:"topics"` } // ListOffsets Response (Version: 0) type ListOffsetsResponsePartitionV0 struct { - PartitionIndex int32 - ErrorCode int16 - OldStyleOffsets int64 + PartitionIndex int32 `json:"partitionIndex"` + ErrorCode int16 `json:"errorCode"` + OldStyleOffsets int64 `json:"oldStyleOffsets"` } type ListOffsetsResponseTopicV0 struct { - Name string - Partitions []ListOffsetsResponsePartitionV0 + Name string `json:"name"` + Partitions []ListOffsetsResponsePartitionV0 `json:"partitions"` } type ListOffsetsResponseV0 struct { - Topics []ListOffsetsResponseTopicV0 + Topics []ListOffsetsResponseTopicV0 `json:"topics"` } // ListOffsets Response (Version: 1) type ListOffsetsResponsePartitionV1 struct { - PartitionIndex int32 - ErrorCode int16 - Timestamp int64 - Offset int64 + PartitionIndex int32 `json:"partitionIndex"` + ErrorCode int16 `json:"errorCode"` + Timestamp int64 `json:"timestamp"` + Offset int64 `json:"offset"` } type ListOffsetsResponseTopicV1 struct { - Name string - Partitions []ListOffsetsResponsePartitionV1 + Name string `json:"name"` + Partitions []ListOffsetsResponsePartitionV1 `json:"partitions"` } type ListOffsetsResponseV1 struct { - Topics []ListOffsetsResponseTopicV1 + Topics []ListOffsetsResponseTopicV1 `json:"topics"` } // ListOffsets Response (Version: 2) type ListOffsetsResponseV2 struct { - ThrottleTimeMs int32 - Topics []ListOffsetsResponseTopicV1 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Topics []ListOffsetsResponseTopicV1 `json:"topics"` } // ListOffsets Response (Version: 4) type ListOffsetsResponsePartitionV4 struct { - PartitionIndex int32 - ErrorCode int16 - Timestamp int64 - Offset int64 - LeaderEpoch int32 + PartitionIndex int32 `json:"partitionIndex"` + ErrorCode int16 `json:"errorCode"` + Timestamp int64 `json:"timestamp"` + Offset int64 `json:"offset"` + LeaderEpoch int32 `json:"leaderEpoch"` } type ListOffsetsResponseTopicV4 struct { - Name string - Partitions []ListOffsetsResponsePartitionV4 + Name string `json:"name"` + Partitions []ListOffsetsResponsePartitionV4 `json:"partitions"` } type ListOffsetsResponseV4 struct { - Topics []ListOffsetsResponseTopicV4 + Topics []ListOffsetsResponseTopicV4 `json:"topics"` } // CreateTopics Request (Version: 0) type AssignmentsV0 struct { - PartitionIndex int32 - BrokerIds []int32 + PartitionIndex int32 `json:"partitionIndex"` + BrokerIds []int32 `json:"brokerIds"` } type CreateTopicsRequestConfigsV0 struct { - Name string - Value string + Name string `json:"name"` + Value string `json:"value"` } type CreateTopicsRequestTopicV0 struct { - Name string - NumPartitions int32 - ReplicationFactor int16 - Assignments []AssignmentsV0 - Configs []CreateTopicsRequestConfigsV0 + Name string `json:"name"` + NumPartitions int32 `json:"numPartitions"` + ReplicationFactor int16 `json:"replicationFactor"` + Assignments []AssignmentsV0 `json:"assignments"` + Configs []CreateTopicsRequestConfigsV0 `json:"configs"` } type CreateTopicsRequestV0 struct { - Topics []CreateTopicsRequestTopicV0 - TimeoutMs int32 + Topics []CreateTopicsRequestTopicV0 `json:"topics"` + TimeoutMs int32 `json:"timeoutMs"` } // CreateTopics Request (Version: 1) type CreateTopicsRequestV1 struct { - Topics []CreateTopicsRequestTopicV0 - TimeoutMs int32 - ValidateOnly bool + Topics []CreateTopicsRequestTopicV0 `json:"topics"` + TimeoutMs int32 `json:"timeoutMs"` + ValidateOnly bool `json:"validateOnly"` } // CreateTopics Response (Version: 0) type CreateTopicsResponseTopicV0 struct { - Name string - ErrorCode int16 + Name string `json:"name"` + ErrorCode int16 `json:"errorCode"` } type CreateTopicsResponseV0 struct { - Topics []CreateTopicsResponseTopicV0 + Topics []CreateTopicsResponseTopicV0 `json:"topics"` } // CreateTopics Response (Version: 1) type CreateTopicsResponseTopicV1 struct { - Name string - ErrorCode int16 - ErrorMessage string + Name string `json:"name"` + ErrorCode int16 `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` } type CreateTopicsResponseV1 struct { - Topics []CreateTopicsResponseTopicV1 + Topics []CreateTopicsResponseTopicV1 `json:"topics"` } // CreateTopics Response (Version: 2) type CreateTopicsResponseV2 struct { - ThrottleTimeMs int32 - Topics []CreateTopicsResponseTopicV1 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Topics []CreateTopicsResponseTopicV1 `json:"topics"` } // CreateTopics Response (Version: 5) type CreateTopicsResponseConfigsV5 struct { - Name string - Value string - ReadOnly bool - ConfigSource int8 - IsSensitive bool + Name string `json:"name"` + Value string `json:"value"` + ReadOnly bool `json:"readOnly"` + ConfigSource int8 `json:"configSource"` + IsSensitive bool `json:"isSensitive"` } type CreateTopicsResponseTopicV5 struct { - Name string - ErrorCode int16 - ErrorMessage string - NumPartitions int32 - ReplicationFactor int16 - Configs []CreateTopicsResponseConfigsV5 + Name string `json:"name"` + ErrorCode int16 `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` + NumPartitions int32 `json:"numPartitions"` + ReplicationFactor int16 `json:"replicationFactor"` + Configs []CreateTopicsResponseConfigsV5 `json:"configs"` } type CreateTopicsResponseV5 struct { - ThrottleTimeMs int32 - Topics []CreateTopicsResponseTopicV5 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Topics []CreateTopicsResponseTopicV5 `json:"topics"` } // CreateTopics Response (Version: 7) type CreateTopicsResponseTopicV7 struct { - Name string - TopicID UUID - ErrorCode int16 - ErrorMessage string - NumPartitions int32 - ReplicationFactor int16 - Configs []CreateTopicsResponseConfigsV5 + Name string `json:"name"` + TopicID UUID `json:"topicID"` + ErrorCode int16 `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` + NumPartitions int32 `json:"numPartitions"` + ReplicationFactor int16 `json:"replicationFactor"` + Configs []CreateTopicsResponseConfigsV5 `json:"configs"` } type CreateTopicsResponseV7 struct { - ThrottleTimeMs int32 - Topics []CreateTopicsResponseTopicV7 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Topics []CreateTopicsResponseTopicV7 `json:"topics"` } // DeleteTopics Request (Version: 0) type DeleteTopicsRequestV0 struct { - TopicNames []string - TimeoutMs int32 + TopicNames []string `json:"topicNames"` + TimeoutMs int32 `json:"timeoutMs"` } // DeleteTopics Request (Version: 6) type DeleteTopicsRequestTopicV6 struct { - Name string - UUID UUID + Name string `json:"name"` + UUID UUID `json:"uuid"` } type DeleteTopicsRequestV6 struct { - Topics []DeleteTopicsRequestTopicV6 - TimeoutMs int32 + Topics []DeleteTopicsRequestTopicV6 `json:"topics"` + TimeoutMs int32 `json:"timeoutMs"` } // DeleteTopics Response (Version: 0) type DeleteTopicsReponseResponseV0 struct { - Name string - ErrorCode int16 + Name string `json:"name"` + ErrorCode int16 `json:"errorCode"` } type DeleteTopicsReponseV0 struct { - Responses []DeleteTopicsReponseResponseV0 + Responses []DeleteTopicsReponseResponseV0 `json:"responses"` } // DeleteTopics Response (Version: 1) type DeleteTopicsReponseV1 struct { - ThrottleTimeMs int32 - Responses []DeleteTopicsReponseResponseV0 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Responses []DeleteTopicsReponseResponseV0 `json:"responses"` } // DeleteTopics Response (Version: 5) type DeleteTopicsReponseResponseV5 struct { - Name string - ErrorCode int16 - ErrorMessage string + Name string `json:"name"` + ErrorCode int16 `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` } type DeleteTopicsReponseV5 struct { - ThrottleTimeMs int32 - Responses []DeleteTopicsReponseResponseV5 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Responses []DeleteTopicsReponseResponseV5 `json:"responses"` } // DeleteTopics Response (Version: 6) type DeleteTopicsReponseResponseV6 struct { - Name string - TopicID UUID - ErrorCode int16 - ErrorMessage string + Name string `json:"name"` + TopicID UUID `json:"topicID"` + ErrorCode int16 `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` } type DeleteTopicsReponseV6 struct { - ThrottleTimeMs int32 - Responses []DeleteTopicsReponseResponseV6 + ThrottleTimeMs int32 `json:"throttleTimeMs"` + Responses []DeleteTopicsReponseResponseV6 `json:"responses"` } diff --git a/tap/extensions/redis/helpers.go b/tap/extensions/redis/helpers.go index a020b722d..3f8b61791 100644 --- a/tap/extensions/redis/helpers.go +++ b/tap/extensions/redis/helpers.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "github.com/up9inc/mizu/tap/api" ) @@ -24,33 +25,38 @@ type RedisWrapper struct { Details interface{} `json:"details"` } -func representGeneric(generic map[string]interface{}) (representation []interface{}) { - details, _ := json.Marshal([]map[string]string{ +func representGeneric(generic map[string]interface{}, selectorPrefix string) (representation []interface{}) { + details, _ := json.Marshal([]api.TableData{ { - "name": "Type", - "value": generic["type"].(string), + Name: "Type", + Value: generic["type"].(string), + Selector: fmt.Sprintf("%stype", selectorPrefix), }, { - "name": "Command", - "value": generic["command"].(string), + Name: "Command", + Value: generic["command"].(string), + Selector: fmt.Sprintf("%scommand", selectorPrefix), }, { - "name": "Key", - "value": generic["key"].(string), + Name: "Key", + Value: generic["key"].(string), + Selector: fmt.Sprintf("%skey", selectorPrefix), }, { - "name": "Value", - "value": generic["value"].(string), + Name: "Value", + Value: generic["value"].(string), + Selector: fmt.Sprintf("%svalue", selectorPrefix), }, { - "name": "Keyword", - "value": generic["keyword"].(string), + Name: "Keyword", + Value: generic["keyword"].(string), + Selector: fmt.Sprintf("%skeyword", selectorPrefix), }, }) - representation = append(representation, map[string]string{ - "type": api.TABLE, - "title": "Details", - "data": string(details), + representation = append(representation, api.SectionData{ + Type: api.TABLE, + Title: "Details", + Data: string(details), }) return diff --git a/tap/extensions/redis/main.go b/tap/extensions/redis/main.go index be0650bfc..387a82b2f 100644 --- a/tap/extensions/redis/main.go +++ b/tap/extensions/redis/main.go @@ -13,6 +13,7 @@ var protocol api.Protocol = api.Protocol{ Name: "redis", LongName: "Redis Serialization Protocol", Abbreviation: "REDIS", + Macro: "redis", Version: "3.x", BackgroundColor: "#a41e11", ForegroundColor: "#ffffff", @@ -57,9 +58,12 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co } } -func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry { +func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry { request := item.Pair.Request.Payload.(map[string]interface{}) + response := item.Pair.Response.Payload.(map[string]interface{}) reqDetails := request["details"].(map[string]interface{}) + resDetails := response["details"].(map[string]interface{}) + service := "redis" if resolvedDestination != "" { service = resolvedDestination @@ -78,45 +82,49 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve } request["url"] = summary - entryBytes, _ := json.Marshal(item.Pair) return &api.MizuEntry{ - ProtocolName: protocol.Name, - ProtocolLongName: protocol.LongName, - ProtocolAbbreviation: protocol.Abbreviation, - ProtocolVersion: protocol.Version, - ProtocolBackgroundColor: protocol.BackgroundColor, - ProtocolForegroundColor: protocol.ForegroundColor, - ProtocolFontSize: protocol.FontSize, - ProtocolReferenceLink: protocol.ReferenceLink, - EntryId: entryId, - Entry: string(entryBytes), - Url: fmt.Sprintf("%s%s", service, summary), - Method: method, - Status: 0, - RequestSenderIp: item.ConnectionInfo.ClientIP, - Service: service, - Timestamp: item.Timestamp, - ElapsedTime: 0, - Path: summary, - ResolvedSource: resolvedSource, - ResolvedDestination: resolvedDestination, - SourceIp: item.ConnectionInfo.ClientIP, - DestinationIp: item.ConnectionInfo.ServerIP, - SourcePort: item.ConnectionInfo.ClientPort, - DestinationPort: item.ConnectionInfo.ServerPort, - IsOutgoing: item.ConnectionInfo.IsOutgoing, + Protocol: protocol, + Source: &api.TCP{ + Name: resolvedSource, + IP: item.ConnectionInfo.ClientIP, + Port: item.ConnectionInfo.ClientPort, + }, + Destination: &api.TCP{ + Name: resolvedDestination, + IP: item.ConnectionInfo.ServerIP, + Port: item.ConnectionInfo.ServerPort, + }, + Outgoing: item.ConnectionInfo.IsOutgoing, + Request: reqDetails, + Response: resDetails, + Url: fmt.Sprintf("%s%s", service, summary), + Method: method, + Status: 0, + RequestSenderIp: item.ConnectionInfo.ClientIP, + Service: service, + Timestamp: item.Timestamp, + StartTime: item.Pair.Request.CaptureTime, + ElapsedTime: 0, + Summary: summary, + ResolvedSource: resolvedSource, + ResolvedDestination: resolvedDestination, + SourceIp: item.ConnectionInfo.ClientIP, + DestinationIp: item.ConnectionInfo.ServerIP, + SourcePort: item.ConnectionInfo.ClientPort, + DestinationPort: item.ConnectionInfo.ServerPort, + IsOutgoing: item.ConnectionInfo.IsOutgoing, } } func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { return &api.BaseEntryDetails{ - Id: entry.EntryId, + Id: entry.Id, Protocol: protocol, Url: entry.Url, RequestSenderIp: entry.RequestSenderIp, Service: entry.Service, - Summary: entry.Path, + Summary: entry.Summary, StatusCode: entry.Status, Method: entry.Method, Timestamp: entry.Timestamp, @@ -133,22 +141,22 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails { } } -func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) { - p = protocol +func (d dissecting) Represent(protoIn api.Protocol, request map[string]interface{}, response map[string]interface{}) (protoOut api.Protocol, object []byte, bodySize int64, err error) { + protoOut = protocol bodySize = 0 - var root map[string]interface{} - json.Unmarshal([]byte(entry.Entry), &root) representation := make(map[string]interface{}, 0) - request := root["request"].(map[string]interface{})["payload"].(map[string]interface{}) - response := root["response"].(map[string]interface{})["payload"].(map[string]interface{}) - reqDetails := request["details"].(map[string]interface{}) - resDetails := response["details"].(map[string]interface{}) - repRequest := representGeneric(reqDetails) - repResponse := representGeneric(resDetails) + repRequest := representGeneric(request, `request.`) + repResponse := representGeneric(response, `response.`) representation["request"] = repRequest representation["response"] = repResponse object, err = json.Marshal(representation) return } +func (d dissecting) Macros() map[string]string { + return map[string]string{ + `redis`: fmt.Sprintf(`proto.abbr == "%s"`, protocol.Abbreviation), + } +} + var Dissector dissecting diff --git a/ui/package-lock.json b/ui/package-lock.json index f6a4eff73..745364d11 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -1769,6 +1769,33 @@ } } }, + "@mapbox/rehype-prism": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@mapbox/rehype-prism/-/rehype-prism-0.7.0.tgz", + "integrity": "sha512-zSG46selA6v+3THhCatTyOt9DuTzxTIVTxTbcj15kFmxPDtjzZ5VoFVCLZfjWFouYa9PiXxcbMLLhJoVzCxh9w==", + "requires": { + "hast-util-to-string": "^1.0.4", + "refractor": "^3.4.0", + "unist-util-visit": "^2.0.3" + }, + "dependencies": { + "prismjs": { + "version": "1.24.1", + "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.24.1.tgz", + "integrity": "sha512-mNPsedLuk90RVJioIky8ANZEwYm5w9LcvCXrxHlwf4fNVSn8jEipMybMkWUyyF0JhnC+C4VcOVSBuHRKs1L5Ow==" + }, + "refractor": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/refractor/-/refractor-3.4.0.tgz", + "integrity": "sha512-dBeD02lC5eytm9Gld2Mx0cMcnR+zhSnsTfPpWqFaMgUMJfC9A6bcN3Br/NaXrnBJcuxnLFR90k1jrkaSyV8umg==", + "requires": { + "hastscript": "^6.0.0", + "parse-entities": "^2.0.0", + "prismjs": "~1.24.0" + } + } + } + }, "@material-ui/core": { "version": "4.12.2", "resolved": "https://registry.npmjs.org/@material-ui/core/-/core-4.12.2.tgz", @@ -1788,6 +1815,14 @@ "react-transition-group": "^4.4.0" } }, + "@material-ui/icons": { + "version": "4.11.2", + "resolved": "https://registry.npmjs.org/@material-ui/icons/-/icons-4.11.2.tgz", + "integrity": "sha512-fQNsKX2TxBmqIGJCSi3tGTO/gZ+eJgWmMJkgDiOfyNaunNaxcklJQFaFogYcFl0qFuaEz1qaXYXboa/bUXVSOQ==", + "requires": { + "@babel/runtime": "^7.4.4" + } + }, "@material-ui/lab": { "version": "4.0.0-alpha.60", "resolved": "https://registry.npmjs.org/@material-ui/lab/-/lab-4.0.0-alpha.60.tgz", @@ -2406,6 +2441,11 @@ "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==" }, + "@types/parse5": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@types/parse5/-/parse5-6.0.1.tgz", + "integrity": "sha512-ARATsLdrGPUnaBvxLhUlnltcMgn7pQG312S8ccdYlnyijabrX9RN/KN/iGj9Am96CoW8e/K9628BA7Bv4XHdrA==" + }, "@types/prettier": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.2.3.tgz", @@ -2629,6 +2669,26 @@ "eslint-visitor-keys": "^2.0.0" } }, + "@uiw/react-textarea-code-editor": { + "version": "1.4.12", + "resolved": "https://registry.npmjs.org/@uiw/react-textarea-code-editor/-/react-textarea-code-editor-1.4.12.tgz", + "integrity": "sha512-op0aIRxX8hLi8OLwm/23dQ2X4o2xHUoK2pLr1DWBFgNbIh4L+RM5zByNFjV8mHfi/NvE6mQLB6LtMd1qaor5MQ==", + "requires": { + "@babel/runtime": "7.15.4", + "@mapbox/rehype-prism": "0.7.0", + "rehype": "12.0.0" + }, + "dependencies": { + "@babel/runtime": { + "version": "7.15.4", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.15.4.tgz", + "integrity": "sha512-99catp6bHCaxr4sJ/DbTGgHS4+Rs2RVd2g7iOap6SLGPDknRK9ztKNsE/Fg6QhSeh1FGE5f6gHGQmvvn3I3xhw==", + "requires": { + "regenerator-runtime": "^0.13.4" + } + } + } + }, "@webassemblyjs/ast": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz", @@ -3689,6 +3749,11 @@ "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.18.0.tgz", "integrity": "sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ==" }, + "bail": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.1.tgz", + "integrity": "sha512-d5FoTAr2S5DSUPKl85WNm2yUwsINN8eidIdIwsOge2t33DaOfOdSmmsI11jMN3GmALCXaw+Y6HMVHDzePshFAA==" + }, "balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -4179,6 +4244,11 @@ "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, + "ccount": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.0.tgz", + "integrity": "sha512-VOR0NWFYX65n9gELQdcpqsie5L5ihBXuZGAgaPEp/U7IOSjnPMEH6geE+2f6lcekaNEfWzAHS45mPvSo5bqsUA==" + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -4199,6 +4269,11 @@ "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.4.tgz", "integrity": "sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw==" }, + "character-entities-html4": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.0.0.tgz", + "integrity": "sha512-dwT2xh5ZhUAjyP96k57ilMKoTQyASaw9IAMR9U5c1lCu2RUni6O6jxfpUEdO2RcPT6TJFvr8pqsbami4Jk+2oA==" + }, "character-entities-legacy": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz", @@ -7534,11 +7609,121 @@ "minimalistic-assert": "^1.0.1" } }, + "hast-util-from-parse5": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-7.1.0.tgz", + "integrity": "sha512-m8yhANIAccpU4K6+121KpPP55sSl9/samzQSQGpb0mTExcNh2WlvjtMwSWFhg6uqD4Rr6Nfa8N6TMypQM51rzQ==", + "requires": { + "@types/hast": "^2.0.0", + "@types/parse5": "^6.0.0", + "@types/unist": "^2.0.0", + "hastscript": "^7.0.0", + "property-information": "^6.0.0", + "vfile": "^5.0.0", + "vfile-location": "^4.0.0", + "web-namespaces": "^2.0.0" + }, + "dependencies": { + "comma-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.2.tgz", + "integrity": "sha512-G5yTt3KQN4Yn7Yk4ed73hlZ1evrFKXeUW3086p3PRFNp7m2vIjI6Pg+Kgb+oyzhd9F2qdcoj67+y3SdxL5XWsg==" + }, + "hast-util-parse-selector": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-3.1.0.tgz", + "integrity": "sha512-AyjlI2pTAZEOeu7GeBPZhROx0RHBnydkQIXlhnFzDi0qfXTmGUWoCYZtomHbrdrheV4VFUlPcfJ6LMF5T6sQzg==", + "requires": { + "@types/hast": "^2.0.0" + } + }, + "hastscript": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-7.0.2.tgz", + "integrity": "sha512-uA8ooUY4ipaBvKcMuPehTAB/YfFLSSzCwFSwT6ltJbocFUKH/GDHLN+tflq7lSRf9H86uOuxOFkh1KgIy3Gg2g==", + "requires": { + "@types/hast": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-parse-selector": "^3.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0" + } + }, + "property-information": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.0.1.tgz", + "integrity": "sha512-F4WUUAF7fMeF4/JUFHNBWDaKDXi2jbvqBW/y6o5wsf3j19wTZ7S60TmtB5HoBhtgw7NKQRMWuz5vk2PR0CygUg==" + }, + "space-separated-tokens": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.1.tgz", + "integrity": "sha512-ekwEbFp5aqSPKaqeY1PGrlGQxPNaq+Cnx4+bE2D8sciBQrHpbwoBbawqTN2+6jPs9IdWxxiUcN0K2pkczD3zmw==" + } + } + }, + "hast-util-is-element": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-2.1.1.tgz", + "integrity": "sha512-ag0fiZfRWsPiR1udvnSbaazJLGv8qd8E+/e3rW8rUZhbKG4HNJmFL4QkEceN+22BgE+uozXY30z/s+2dL6Z++g==", + "requires": { + "@types/hast": "^2.0.0", + "@types/unist": "^2.0.0" + } + }, "hast-util-parse-selector": { "version": "2.2.5", "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz", "integrity": "sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ==" }, + "hast-util-to-html": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-8.0.2.tgz", + "integrity": "sha512-ipLhUTMyyJi9F/LXaNDG9BrRdshP6obCfmUZYbE/+T639IdzqAOkKN4DyrEyID0gbb+rsC3PKf0XlviZwzomhw==", + "requires": { + "@types/hast": "^2.0.0", + "ccount": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-is-element": "^2.0.0", + "hast-util-whitespace": "^2.0.0", + "html-void-elements": "^2.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "stringify-entities": "^4.0.0", + "unist-util-is": "^5.0.0" + }, + "dependencies": { + "comma-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.2.tgz", + "integrity": "sha512-G5yTt3KQN4Yn7Yk4ed73hlZ1evrFKXeUW3086p3PRFNp7m2vIjI6Pg+Kgb+oyzhd9F2qdcoj67+y3SdxL5XWsg==" + }, + "property-information": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.0.1.tgz", + "integrity": "sha512-F4WUUAF7fMeF4/JUFHNBWDaKDXi2jbvqBW/y6o5wsf3j19wTZ7S60TmtB5HoBhtgw7NKQRMWuz5vk2PR0CygUg==" + }, + "space-separated-tokens": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.1.tgz", + "integrity": "sha512-ekwEbFp5aqSPKaqeY1PGrlGQxPNaq+Cnx4+bE2D8sciBQrHpbwoBbawqTN2+6jPs9IdWxxiUcN0K2pkczD3zmw==" + }, + "unist-util-is": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-5.1.1.tgz", + "integrity": "sha512-F5CZ68eYzuSvJjGhCLPL3cYx45IxkqXSetCcRgUXtbcm50X2L9oOWQlfUfDdAf+6Pd27YDblBfdtmsThXmwpbQ==" + } + } + }, + "hast-util-to-string": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/hast-util-to-string/-/hast-util-to-string-1.0.4.tgz", + "integrity": "sha512-eK0MxRX47AV2eZ+Lyr18DCpQgodvaS3fAQO2+b9Two9F5HEoRPhiUMNzoXArMJfZi2yieFzUBMRl3HNJ3Jus3w==" + }, + "hast-util-whitespace": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-2.0.0.tgz", + "integrity": "sha512-Pkw+xBHuV6xFeJprJe2BBEoDV+AvQySaz3pPDRUs5PNZEMQjpXJJueqrpcHIXxnWTcAGi/UOCgVShlkY6kLoqg==" + }, "hastscript": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-6.0.0.tgz", @@ -7671,6 +7856,11 @@ "terser": "^4.6.3" } }, + "html-void-elements": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-2.0.0.tgz", + "integrity": "sha512-4OYzQQsBt0G9bJ/nM9/DDsjm4+fVdzAaPJJcWk5QwA3GIAPxQEeOR0rsI8HbDHQz5Gta8pVvGnnTNSbZVEVvkQ==" + }, "html-webpack-plugin": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-4.5.0.tgz", @@ -13470,6 +13660,14 @@ "refractor": "^3.2.0" } }, + "react-toastify": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/react-toastify/-/react-toastify-8.0.3.tgz", + "integrity": "sha512-rv3koC7f9lKKSkdpYgo/TGzgWlrB/aaiUInF1DyV7BpiM4kyTs+uhu6/r8XDMtBY2FOIHK+FlK3Iv7OzpA/tCA==", + "requires": { + "clsx": "^1.1.1" + } + }, "react-transition-group": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.1.tgz", @@ -13693,6 +13891,38 @@ } } }, + "rehype": { + "version": "12.0.0", + "resolved": "https://registry.npmjs.org/rehype/-/rehype-12.0.0.tgz", + "integrity": "sha512-gZcttmf9R5IYHb8AlI1rlmWqXS1yX0rSB/S5ZGJs8atfYZy2DobvH3Ic/gSzB+HL/+oOHPtBguw1TprfhxXBgQ==", + "requires": { + "@types/hast": "^2.0.0", + "rehype-parse": "^8.0.0", + "rehype-stringify": "^9.0.0", + "unified": "^10.0.0" + } + }, + "rehype-parse": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/rehype-parse/-/rehype-parse-8.0.3.tgz", + "integrity": "sha512-RGw0CVt+0S6KdvpE8bbP2Db9WXclQcIX7A0ufM3QFqAhTo/ddJMQrrI2j3cijlRPZlGK8R3pRgC8U5HyV76IDw==", + "requires": { + "@types/hast": "^2.0.0", + "hast-util-from-parse5": "^7.0.0", + "parse5": "^6.0.0", + "unified": "^10.0.0" + } + }, + "rehype-stringify": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/rehype-stringify/-/rehype-stringify-9.0.2.tgz", + "integrity": "sha512-BuVA6lAEYtOpXO2xuHLohAzz8UNoQAxAqYRqh4QEEtU39Co+P1JBZhw6wXA9hMWp+JLcmrxWH8+UKcNSr443Fw==", + "requires": { + "@types/hast": "^2.0.0", + "hast-util-to-html": "^8.0.0", + "unified": "^10.0.0" + } + }, "relateurl": { "version": "0.2.7", "resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz", @@ -15342,6 +15572,22 @@ } } }, + "stringify-entities": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.1.tgz", + "integrity": "sha512-gmMQxKXPWIO3NXNSPyWNhlYcBNGpPA/487D+9dLPnU4xBnIrnHdr8cv5rGJOS/1BRxEXRb7uKwg7BA36IWV7xg==", + "requires": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^2.0.0" + }, + "dependencies": { + "character-entities-legacy": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-2.0.0.tgz", + "integrity": "sha512-YwaEtEvWLpFa6Wh3uVLrvirA/ahr9fki/NUd/Bd4OR6EdJ8D22hovYQEOUCBfQfcqnC4IAMGMsHXY1eXgL4ZZA==" + } + } + }, "stringify-object": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz", @@ -15877,6 +16123,11 @@ "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz", "integrity": "sha1-WIeWa7WCpFA6QetST301ARgVphM=" }, + "trough": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.0.2.tgz", + "integrity": "sha512-FnHq5sTMxC0sk957wHDzRnemFnNBvt/gSY99HzK8F7UP5WAbvP70yX5bd7CjEQkN+TjdxwI7g7lJ6podqrG2/w==" + }, "true-case-path": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/true-case-path/-/true-case-path-1.0.3.tgz", @@ -16037,6 +16288,32 @@ "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-1.1.0.tgz", "integrity": "sha512-PqSoPh/pWetQ2phoj5RLiaqIk4kCNwoV3CI+LfGmWLKI3rE3kl1h59XpX2BjgDrmbxD9ARtQobPGU1SguCYuQg==" }, + "unified": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/unified/-/unified-10.1.0.tgz", + "integrity": "sha512-4U3ru/BRXYYhKbwXV6lU6bufLikoAavTwev89H5UxY8enDFaAT2VXmIXYNm6hb5oHPng/EXr77PVyDFcptbk5g==", + "requires": { + "@types/unist": "^2.0.0", + "bail": "^2.0.0", + "extend": "^3.0.0", + "is-buffer": "^2.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^5.0.0" + }, + "dependencies": { + "is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" + }, + "is-plain-obj": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.0.0.tgz", + "integrity": "sha512-NXRbBtUdBioI73y/HmOhogw/U5msYPC9DAtGkJXeFcFWSFZw0mCUsPxk/snTuJHzNKA8kLBK4rH97RMB1BfCXw==" + } + } + }, "union-value": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", @@ -16082,6 +16359,38 @@ "crypto-random-string": "^1.0.0" } }, + "unist-util-is": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz", + "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==" + }, + "unist-util-stringify-position": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-3.0.0.tgz", + "integrity": "sha512-SdfAl8fsDclywZpfMDTVDxA2V7LjtRDTOFd44wUJamgl6OlVngsqWjxvermMYf60elWHbxhuRCZml7AnuXCaSA==", + "requires": { + "@types/unist": "^2.0.0" + } + }, + "unist-util-visit": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-2.0.3.tgz", + "integrity": "sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0", + "unist-util-visit-parents": "^3.0.0" + } + }, + "unist-util-visit-parents": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz", + "integrity": "sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0" + } + }, "universalify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", @@ -16306,6 +16615,42 @@ "extsprintf": "^1.2.0" } }, + "vfile": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-5.1.0.tgz", + "integrity": "sha512-4o7/DJjEaFPYSh0ckv5kcYkJTHQgCKdL8ozMM1jLAxO9ox95IzveDPXCZp08HamdWq8JXTkClDvfAKaeLQeKtg==", + "requires": { + "@types/unist": "^2.0.0", + "is-buffer": "^2.0.0", + "unist-util-stringify-position": "^3.0.0", + "vfile-message": "^3.0.0" + }, + "dependencies": { + "is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" + } + } + }, + "vfile-location": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-4.0.1.tgz", + "integrity": "sha512-JDxPlTbZrZCQXogGheBHjbRWjESSPEak770XwWPfw5mTc1v1nWGLB/apzZxsx8a0SJVfF8HK8ql8RD308vXRUw==", + "requires": { + "@types/unist": "^2.0.0", + "vfile": "^5.0.0" + } + }, + "vfile-message": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-3.0.2.tgz", + "integrity": "sha512-UUjZYIOg9lDRwwiBAuezLIsu9KlXntdxwG+nXnjuQAHvBpcX3x0eN8h+I7TkY5nkCXj+cWVp4ZqebtGBvok8ww==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-stringify-position": "^3.0.0" + } + }, "vm-browserify": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", @@ -16590,6 +16935,11 @@ "minimalistic-assert": "^1.0.0" } }, + "web-namespaces": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.0.tgz", + "integrity": "sha512-dE7ELZRVWh0ceQsRgkjLgsAvwTuv3kcjSY/hLjqL0llleUlQBDjE9JkB9FCBY5F2mnFEwiyJoowl8+NVGHe8dw==" + }, "web-vitals": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/web-vitals/-/web-vitals-1.1.1.tgz", diff --git a/ui/package.json b/ui/package.json index 463ef8c4b..088241fe0 100644 --- a/ui/package.json +++ b/ui/package.json @@ -4,6 +4,7 @@ "private": true, "dependencies": { "@material-ui/core": "^4.11.3", + "@material-ui/icons": "^4.11.2", "@material-ui/lab": "^4.0.0-alpha.60", "@testing-library/jest-dom": "^5.11.10", "@testing-library/react": "^11.2.6", @@ -12,6 +13,7 @@ "@types/node": "^12.20.10", "@types/react": "^17.0.3", "@types/react-dom": "^17.0.3", + "@uiw/react-textarea-code-editor": "^1.4.12", "axios": "^0.21.1", "jsonpath": "^1.1.1", "node-sass": "^5.0.0", @@ -23,6 +25,7 @@ "react-scripts": "4.0.3", "react-scrollable-feed-virtualized": "^1.4.3", "react-syntax-highlighter": "^15.4.3", + "react-toastify": "^8.0.3", "typescript": "^4.2.4", "web-vitals": "^1.1.1" }, diff --git a/ui/src/components/EntriesList.tsx b/ui/src/components/EntriesList.tsx index 53e653a05..4fc05895e 100644 --- a/ui/src/components/EntriesList.tsx +++ b/ui/src/components/EntriesList.tsx @@ -1,10 +1,7 @@ import {EntryItem} from "./EntryListItem/EntryListItem"; -import React, {useCallback, useEffect, useMemo, useRef, useState} from "react"; +import React, {useRef} from "react"; import styles from './style/EntriesList.module.sass'; -import spinner from './assets/spinner.svg'; import ScrollableFeedVirtualized from "react-scrollable-feed-virtualized"; -import {StatusType} from "./Filters"; -import Api from "../helpers/api"; import down from "./assets/downImg.svg"; interface EntriesListProps { @@ -13,114 +10,36 @@ interface EntriesListProps { focusedEntryId: string; setFocusedEntryId: (id: string) => void; connectionOpen: boolean; - noMoreDataTop: boolean; - setNoMoreDataTop: (flag: boolean) => void; - noMoreDataBottom: boolean; - setNoMoreDataBottom: (flag: boolean) => void; - methodsFilter: Array; - statusFilter: Array; - pathFilter: string; - serviceFilter: string; listEntryREF: any; onScrollEvent: (isAtBottom:boolean) => void; scrollableList: boolean; + ws: any + openWebSocket: any; + query: string; + updateQuery: any; + queriedCurrent: number; + queriedTotal: number; + startTime: number; } -enum FetchOperator { - LT = "lt", - GT = "gt" -} - -const api = new Api(); - -export const EntriesList: React.FC = ({entries, setEntries, focusedEntryId, setFocusedEntryId, connectionOpen, noMoreDataTop, setNoMoreDataTop, noMoreDataBottom, setNoMoreDataBottom, methodsFilter, statusFilter, pathFilter, serviceFilter, listEntryREF, onScrollEvent, scrollableList}) => { - - const [loadMoreTop, setLoadMoreTop] = useState(false); - const [isLoadingTop, setIsLoadingTop] = useState(false); +export const EntriesList: React.FC = ({entries, setEntries, focusedEntryId, setFocusedEntryId, connectionOpen, listEntryREF, onScrollEvent, scrollableList, ws, openWebSocket, query, updateQuery, queriedCurrent, queriedTotal, startTime}) => { const scrollableRef = useRef(null); - useEffect(() => { - const list = document.getElementById('list').firstElementChild; - list.addEventListener('scroll', (e) => { - const el: any = e.target; - if(el.scrollTop === 0) { - setLoadMoreTop(true); - } else { - setLoadMoreTop(false); - } - }); - }, []); - - const filterEntries = useCallback((entry) => { - if(methodsFilter.length > 0 && !methodsFilter.includes(entry.method.toLowerCase())) return; - if(pathFilter && entry.path?.toLowerCase()?.indexOf(pathFilter) === -1) return; - if(serviceFilter && entry.service?.toLowerCase()?.indexOf(serviceFilter) === -1) return; - if(statusFilter.includes(StatusType.SUCCESS) && entry.statusCode >= 400) return; - if(statusFilter.includes(StatusType.ERROR) && entry.statusCode < 400) return; - return entry; - },[methodsFilter, pathFilter, statusFilter, serviceFilter]) - - const filteredEntries = useMemo(() => { - return entries.filter(filterEntries); - },[entries, filterEntries]) - - const getOldEntries = useCallback(async () => { - setIsLoadingTop(true); - const data = await api.fetchEntries(FetchOperator.LT, entries[0].timestamp); - setLoadMoreTop(false); - - let scrollTo; - if(data.length === 0) { - setNoMoreDataTop(true); - scrollTo = document.getElementById("noMoreDataTop"); - } else { - scrollTo = document.getElementById(filteredEntries?.[0]?.id); - } - setIsLoadingTop(false); - const newEntries = [...data, ...entries]; - setEntries(newEntries); - - if(scrollTo) { - scrollTo.scrollIntoView(); - } - },[setLoadMoreTop, setIsLoadingTop, entries, setEntries, filteredEntries, setNoMoreDataTop]) - - useEffect(() => { - if(!loadMoreTop || connectionOpen || noMoreDataTop) return; - getOldEntries(); - }, [loadMoreTop, connectionOpen, noMoreDataTop, getOldEntries]); - - const getNewEntries = async () => { - const data = await api.fetchEntries(FetchOperator.GT, entries[entries.length - 1].timestamp); - let scrollTo; - if(data.length === 0) { - setNoMoreDataBottom(true); - } - scrollTo = document.getElementById(filteredEntries?.[filteredEntries.length -1]?.id); - let newEntries = [...entries, ...data]; - setEntries(newEntries); - if(scrollTo) { - scrollTo.scrollIntoView({behavior: "smooth"}); - } - } - return <>
- {isLoadingTop &&
- spinner -
} onScrollEvent(isAtBottom)}> - {noMoreDataTop && !connectionOpen &&
No more data available
} - {filteredEntries.map(entry => )} + isSelected={focusedEntryId === entry.id.toString()} + style={{}} + updateQuery={updateQuery}/>)} - {!connectionOpen && !noMoreDataBottom &&
-
getNewEntries()}>Fetch more entries
+ {!connectionOpen &&
+
{ws.close(); openWebSocket(query);}}>Reconnect
}
- {entries?.length > 0 &&
-
{filteredEntries?.length !== entries.length && `${filteredEntries?.length} / `} {entries?.length} requests
-
Started listening at {new Date(+entries[0].timestamp)?.toLocaleString()}
-
} +
+
Displaying {entries?.length} results (queried {queriedCurrent}/{queriedTotal})
+ {startTime !== 0 &&
Started listening at {new Date(startTime).toLocaleString()}
} +
; }; diff --git a/ui/src/components/EntryDetailed.tsx b/ui/src/components/EntryDetailed.tsx index ffc5151a9..2c5481a13 100644 --- a/ui/src/components/EntryDetailed.tsx +++ b/ui/src/components/EntryDetailed.tsx @@ -3,7 +3,7 @@ import EntryViewer from "./EntryDetailed/EntryViewer"; import {makeStyles} from "@material-ui/core"; import Protocol from "./UI/Protocol" import StatusCode from "./UI/StatusCode"; -import {EndpointPath} from "./UI/EndpointPath"; +import {Summary} from "./UI/Summary"; const useStyles = makeStyles(() => ({ entryTitle: { @@ -28,50 +28,79 @@ const useStyles = makeStyles(() => ({ interface EntryDetailedProps { entryData: any + updateQuery: any } export const formatSize = (n: number) => n > 1000 ? `${Math.round(n / 1000)}KB` : `${n} B`; -const EntryTitle: React.FC = ({protocol, data, bodySize, elapsedTime}) => { +const EntryTitle: React.FC = ({protocol, data, bodySize, elapsedTime, updateQuery}) => { const classes = useStyles(); - const {response} = JSON.parse(data.entry); + const response = data.response; return
- +
- {response.payload &&
{formatSize(bodySize)}
} - {response.payload &&
{Math.round(elapsedTime)}ms
} + {response &&
{ + updateQuery(`response.bodySize == ${bodySize}`) + }} + > + {formatSize(bodySize)} +
} + {response &&
{ + updateQuery(`elapsedTime >= ${elapsedTime}`) + }} + > + {Math.round(elapsedTime)}ms +
}
; }; -const EntrySummary: React.FC = ({data}) => { +const EntrySummary: React.FC = ({data, updateQuery}) => { const classes = useStyles(); - const {response, request} = JSON.parse(data.entry); + const response = data.response; return
- {response?.payload && response.payload?.details && "status" in response.payload.details &&
- + {response && "status" in response &&
+
}
- +
; }; -export const EntryDetailed: React.FC = ({entryData}) => { +export const EntryDetailed: React.FC = ({entryData, updateQuery}) => { return <> - {entryData.data && } + {entryData.data && } <> - {entryData.data && } + {entryData.data && } }; diff --git a/ui/src/components/EntryDetailed/EntrySections.tsx b/ui/src/components/EntryDetailed/EntrySections.tsx index f364b4a51..b702b4db1 100644 --- a/ui/src/components/EntryDetailed/EntrySections.tsx +++ b/ui/src/components/EntryDetailed/EntrySections.tsx @@ -9,11 +9,29 @@ import ProtobufDecoder from "protobuf-decoder"; interface EntryViewLineProps { label: string; value: number | string; + updateQuery: any; + selector: string; + overrideQueryValue?: string; } -const EntryViewLine: React.FC = ({label, value}) => { +const EntryViewLine: React.FC = ({label, value, updateQuery, selector, overrideQueryValue}) => { return (label && value && - {label} + { + if (!selector) { + return + } else if (overrideQueryValue) { + updateQuery(`${selector} == ${overrideQueryValue}`) + } else if (typeof(value) === "string") { + updateQuery(`${selector} == "${JSON.stringify(value).slice(1, -1)}"`) + } else { + updateQuery(`${selector} == ${value}`) + } + }} + > + {label} + = ({tit interface EntryBodySectionProps { content: any, color: string, + updateQuery: any, encoding?: string, contentType?: string, + selector?: string, } export const EntryBodySection: React.FC = ({ color, + updateQuery, content, encoding, contentType, + selector, }) => { const MAXIMUM_BYTES_TO_HIGHLIGHT = 10000; // The maximum of chars to highlight in body, in case the response can be megabytes const supportedLanguages = [['html', 'html'], ['json', 'json'], ['application/grpc', 'json']]; // [[indicator, languageToUse],...] @@ -107,8 +129,8 @@ export const EntryBodySection: React.FC = ({ {content && content?.length > 0 && - - + +
@@ -132,17 +154,23 @@ interface EntrySectionProps { title: string, color: string, arrayToIterate: any[], + updateQuery: any, } -export const EntryTableSection: React.FC = ({title, color, arrayToIterate}) => { +export const EntryTableSection: React.FC = ({title, color, arrayToIterate, updateQuery}) => { return { arrayToIterate && arrayToIterate.length > 0 ? - {arrayToIterate.map(({name, value}, index) => )} + {arrayToIterate.map(({name, value, selector}, index) => )}
: diff --git a/ui/src/components/EntryDetailed/EntryViewer.tsx b/ui/src/components/EntryDetailed/EntryViewer.tsx index dc8b8f4c7..dabf3d5b7 100644 --- a/ui/src/components/EntryDetailed/EntryViewer.tsx +++ b/ui/src/components/EntryDetailed/EntryViewer.tsx @@ -8,7 +8,7 @@ enum SectionTypes { SectionBody = "body", } -const SectionsRepresentation: React.FC = ({data, color}) => { +const SectionsRepresentation: React.FC = ({data, color, updateQuery}) => { const sections = [] if (data) { @@ -16,12 +16,12 @@ const SectionsRepresentation: React.FC = ({data, color}) => { switch (row.type) { case SectionTypes.SectionTable: sections.push( - + ) break; case SectionTypes.SectionBody: sections.push( - + ) break; default: @@ -33,7 +33,7 @@ const SectionsRepresentation: React.FC = ({data, color}) => { return <>{sections}; } -const AutoRepresentation: React.FC = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color}) => { +const AutoRepresentation: React.FC = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color, updateQuery}) => { var TABS = [ { tab: 'Request' @@ -85,10 +85,10 @@ const AutoRepresentation: React.FC = ({representation, isRulesEnabled, rule
{currentTab === TABS[0].tab && - + } {response && currentTab === TABS[responseTabIndex].tab && - + } {isRulesEnabled && currentTab === TABS[rulesTabIndex].tab && @@ -110,9 +110,10 @@ interface Props { contractContent: string; color: string; elapsedTime: number; + updateQuery: any; } -const EntryViewer: React.FC = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color}) => { +const EntryViewer: React.FC = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color, updateQuery}) => { return = ({representation, isRulesEnabled, rulesMatc contractContent={contractContent} elapsedTime={elapsedTime} color={color} + updateQuery={updateQuery} /> }; diff --git a/ui/src/components/EntryListItem/EntryListItem.tsx b/ui/src/components/EntryListItem/EntryListItem.tsx index fc418631f..ae771801d 100644 --- a/ui/src/components/EntryListItem/EntryListItem.tsx +++ b/ui/src/components/EntryListItem/EntryListItem.tsx @@ -2,7 +2,7 @@ import React from "react"; import styles from './EntryListItem.module.sass'; import StatusCode, {getClassification, StatusCodeClassification} from "../UI/StatusCode"; import Protocol, {ProtocolInterface} from "../UI/Protocol" -import {EndpointPath} from "../UI/EndpointPath"; +import {Summary} from "../UI/Summary"; import ingoingIconSuccess from "../assets/ingoing-traffic-success.svg" import ingoingIconFailure from "../assets/ingoing-traffic-failure.svg" import ingoingIconNeutral from "../assets/ingoing-traffic-neutral.svg" @@ -15,7 +15,7 @@ interface Entry { method?: string, summary: string, service: string, - id: string, + id: number, statusCode?: number; url?: string; timestamp: Date; @@ -40,9 +40,10 @@ interface EntryProps { setFocusedEntryId: (id: string) => void; isSelected?: boolean; style: object; + updateQuery: any; } -export const EntryItem: React.FC = ({entry, setFocusedEntryId, isSelected, style}) => { +export const EntryItem: React.FC = ({entry, setFocusedEntryId, isSelected, style, updateQuery}) => { const classification = getClassification(entry.statusCode) const numberOfRules = entry.rules.numberOfRules let ingoingIcon; @@ -115,10 +116,10 @@ export const EntryItem: React.FC = ({entry, setFocusedEntryId, isSel return <>
setFocusedEntryId(entry.id)} + onClick={() => setFocusedEntryId(entry.id.toString())} style={{ border: isSelected ? `1px ${entry.protocol.backgroundColor} solid` : "1px transparent solid", position: "absolute", @@ -127,14 +128,26 @@ export const EntryItem: React.FC = ({entry, setFocusedEntryId, isSel width: "calc(100% - 25px)", }} > - + {((entry.protocol.name === "http" && "statusCode" in entry) || entry.statusCode !== 0) &&
- +
}
- +
- {entry.service} + { + updateQuery(`service == "${entry.service}"`) + }} + > + {entry.service} +
{ @@ -152,17 +165,53 @@ export const EntryItem: React.FC = ({entry, setFocusedEntryId, isSel : "" }
- {entry.sourcePort} + { + updateQuery(`src.port == "${entry.sourcePort}"`) + }} + > + {entry.sourcePort} + {entry.isOutgoing ? - Ingoing traffic + Ingoing traffic { + updateQuery(`outgoing == true`) + }} + /> : - Outgoing traffic + Outgoing traffic { + updateQuery(`outgoing == false`) + }} + /> } - {entry.destinationPort} + { + updateQuery(`dst.port == "${entry.destinationPort}"`) + }} + > + {entry.destinationPort} +
- - {new Date(+entry.timestamp)?.toLocaleString()} + { + updateQuery(`timestamp >= datetime("${new Date(+entry.timestamp)?.toLocaleString("en-US", {timeZone: 'UTC' })}")`) + }} + > + {new Date(+entry.timestamp)?.toLocaleString("en-US")}
diff --git a/ui/src/components/Filters.tsx b/ui/src/components/Filters.tsx index c1795a67a..3c7e38f74 100644 --- a/ui/src/components/Filters.tsx +++ b/ui/src/components/Filters.tsx @@ -1,137 +1,303 @@ -import React from "react"; +import React, {useRef, useState} from "react"; import styles from './style/Filters.module.sass'; -import {FilterSelect} from "./UI/FilterSelect"; -import {TextField} from "@material-ui/core"; -import {ALL_KEY} from "./UI/Select"; +import {Button, Grid, Modal, Box, Typography, Backdrop, Fade, Divider} from "@material-ui/core"; +import CodeEditor from '@uiw/react-textarea-code-editor'; +import MenuBookIcon from '@material-ui/icons/MenuBook'; +import {SyntaxHighlighter} from "./UI/SyntaxHighlighter/index"; +import filterUIExample1 from "./assets/filter-ui-example-1.png" +import filterUIExample2 from "./assets/filter-ui-example-2.png" interface FiltersProps { - methodsFilter: Array; - setMethodsFilter: (methods: Array) => void; - statusFilter: Array; - setStatusFilter: (methods: Array) => void; - pathFilter: string - setPathFilter: (val: string) => void; - serviceFilter: string - setServiceFilter: (val: string) => void; + query: string + setQuery: any + backgroundColor: string + ws: any + openWebSocket: (query: string) => void; } -export const Filters: React.FC = ({methodsFilter, setMethodsFilter, statusFilter, setStatusFilter, pathFilter, setPathFilter, serviceFilter, setServiceFilter}) => { - +export const Filters: React.FC = ({query, setQuery, backgroundColor, ws, openWebSocket}) => { return
- - - - +
; }; -const _toUpperCase = v => v.toUpperCase(); +interface QueryFormProps { + query: string + setQuery: any + backgroundColor: string + ws: any + openWebSocket: (query: string) => void; +} -const FilterContainer: React.FC = ({children}) => { - return
- {children} -
; +const style = { + position: 'absolute', + top: '50%', + left: '50%', + transform: 'translate(-50%, -50%)', + width: '80vw', + bgcolor: 'background.paper', + borderRadius: '5px', + boxShadow: 24, + p: 4, + color: '#000', }; -enum HTTPMethod { - GET = "get", - PUT = "put", - POST = "post", - DELETE = "delete", - OPTIONS="options", - PATCH = "patch" -} +export const QueryForm: React.FC = ({query, setQuery, backgroundColor, ws, openWebSocket}) => { -interface MethodFilterProps { - methodsFilter: Array; - setMethodsFilter: (methods: Array) => void; -} + const formRef = useRef(null); -const MethodFilter: React.FC = ({methodsFilter, setMethodsFilter}) => { + const [openModal, setOpenModal] = useState(false); - const methodClicked = (val) => { - if(val === ALL_KEY) { - setMethodsFilter([]); - return; - } - if(methodsFilter.includes(val)) { - setMethodsFilter(methodsFilter.filter(method => method !== val)) - } else { - setMethodsFilter([...methodsFilter, val]); - } + const handleOpenModal = () => setOpenModal(true); + const handleCloseModal = () => setOpenModal(false); + + const handleChange = async (e) => { + setQuery(e.target.value); } - return - methodClicked(val)} - transformDisplay={_toUpperCase} - label={"Methods"} - /> - ; -}; - -export enum StatusType { - SUCCESS = "success", - ERROR = "error" -} - -interface StatusTypesFilterProps { - statusFilter: Array; - setStatusFilter: (methods: Array) => void; -} - -const StatusTypesFilter: React.FC = ({statusFilter, setStatusFilter}) => { - - const statusClicked = (val) => { - if(val === ALL_KEY) { - setStatusFilter([]); - return; - } - setStatusFilter([val]); + const handleSubmit = (e) => { + ws.close() + openWebSocket(query) + e.preventDefault(); } - return - statusClicked(val)} - transformDisplay={_toUpperCase} - label="Status" - /> - ; -}; + return <> +
+ + + + + + + + + +
-interface PathFilterProps { - pathFilter: string; - setPathFilter: (val: string) => void; + + + + + Filtering Guide (Cheatsheet) + + +

Mizu has a rich filtering syntax that let's you query the results both flexibly and efficiently.

+

Here are some examples that you can try;

+ + + + + This is a simple query that matches to HTTP packets with request path "/catalogue": + + + + The same query can be negated for HTTP path and written like this: + + + + The syntax supports regular expressions. Here is a query that matches the HTTP requests that send JSON to a server: + + + + Here is another query that matches HTTP responses with status code 4xx: + + + + The same exact query can be as integer comparison: + + = 400`} + language="python" + /> + + The results can be queried based on their timestamps: + + + + + + + Since Mizu supports various protocols like gRPC, AMQP, Kafka and Redis. It's possible to write complex queries that match multiple protocols like this: + + + + By clicking the UI elements in both left-pane and right-pane, you can automatically select a field and update the query: + + Clicking to UI elements (left-pane) + + Such that; clicking this in left-pane, would append the query below: + + + + Another queriable UI element example, this time from the right-pane: + + Clicking to UI elements (right-pane) + + A query that compares one selector to another is also a valid query: + + + + + + + There are a few helper methods included the in the filter language* to help building queries more easily. + +

+ + true if the given selector's value starts with the string: + + + + true if the given selector's value ends with the string: + + + + true if the given selector's value contains the string: + + + + returns the UNIX timestamp which is the equivalent of the time that's provided by the string. Invalid input evaluates to false: + + = datetime("10/19/2021, 6:29:02 PM")`} + language="python" + /> + + limits the number of records that are streamed back as a result of a query. Always evaluates to true: + + + + +

+ + *The filtering functionality is provided through Basenine database server. Please refer to BFL Syntax Reference for more information. + + + + + } - -const PathFilter: React.FC = ({pathFilter, setPathFilter}) => { - - return -
Path
-
- setPathFilter(e.target.value)}/> -
- ; -}; - -interface ServiceFilterProps { - serviceFilter: string; - setServiceFilter: (val: string) => void; -} - -const ServiceFilter: React.FC = ({serviceFilter, setServiceFilter}) => { - - return -
Service
-
- setServiceFilter(e.target.value)}/> -
- ; -}; - diff --git a/ui/src/components/TrafficPage.tsx b/ui/src/components/TrafficPage.tsx index f1e3be2b1..b8ea48ff7 100644 --- a/ui/src/components/TrafficPage.tsx +++ b/ui/src/components/TrafficPage.tsx @@ -10,6 +10,8 @@ import pauseIcon from './assets/pause.svg'; import variables from '../variables.module.scss'; import {StatusBar} from "./UI/StatusBar"; import Api, {MizuWebsocketURL} from "../helpers/api"; +import { ToastContainer, toast } from 'react-toastify'; +import 'react-toastify/dist/ReactToastify.css'; const useLayoutStyles = makeStyles(() => ({ details: { @@ -34,7 +36,6 @@ const useLayoutStyles = makeStyles(() => ({ enum ConnectionStatus { Closed, Connected, - Paused } interface TrafficPageProps { @@ -52,25 +53,52 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS const [focusedEntryId, setFocusedEntryId] = useState(null); const [selectedEntryData, setSelectedEntryData] = useState(null); const [connection, setConnection] = useState(ConnectionStatus.Closed); - const [noMoreDataTop, setNoMoreDataTop] = useState(false); - const [noMoreDataBottom, setNoMoreDataBottom] = useState(false); - - const [methodsFilter, setMethodsFilter] = useState([]); - const [statusFilter, setStatusFilter] = useState([]); - const [pathFilter, setPathFilter] = useState(""); - const [serviceFilter, setServiceFilter] = useState(""); const [tappingStatus, setTappingStatus] = useState(null); const [disableScrollList, setDisableScrollList] = useState(false); + const [query, setQueryDefault] = useState(""); + const [queryBackgroundColor, setQueryBackgroundColor] = useState("#f5f5f5"); + + const [queriedCurrent, setQueriedCurrent] = useState(0); + const [queriedTotal, setQueriedTotal] = useState(0); + + const [startTime, setStartTime] = useState(0); + + const setQuery = async (query) => { + if (!query) { + setQueryBackgroundColor("#f5f5f5") + } else { + const data = await api.validateQuery(query); + if (data.valid) { + setQueryBackgroundColor("#d2fad2") + } else { + setQueryBackgroundColor("#fad6dc") + } + } + setQueryDefault(query) + } + + const updateQuery = (addition) => { + if (query) { + setQuery(`${query} and ${addition}`) + } else { + setQuery(addition) + } + } + const ws = useRef(null); const listEntry = useRef(null); - const openWebSocket = () => { + const openWebSocket = (query) => { + setEntries([]) ws.current = new WebSocket(MizuWebsocketURL); - ws.current.onopen = () => setConnection(ConnectionStatus.Connected); + ws.current.onopen = () => { + ws.current.send(query) + setConnection(ConnectionStatus.Connected); + } ws.current.onclose = () => setConnection(ConnectionStatus.Closed); } @@ -81,11 +109,7 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS switch (message.messageType) { case "entry": const entry = message.data - if (connection === ConnectionStatus.Paused) { - setNoMoreDataBottom(false) - return; - } - if (!focusedEntryId) setFocusedEntryId(entry.id) + if (!focusedEntryId) setFocusedEntryId(entry.id.toString()) let newEntries = [...entries]; setEntries([...newEntries, entry]) if(listEntry.current) { @@ -103,6 +127,25 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS case "outboundLink": onTLSDetected(message.Data.DstIP); break; + case "toast": + toast[message.data.type](message.data.text, { + position: "bottom-right", + theme: "colored", + autoClose: message.data.autoClose, + hideProgressBar: false, + closeOnClick: true, + pauseOnHover: true, + draggable: true, + progress: undefined, + }); + break; + case "queryMetadata": + setQueriedCurrent(message.data.current) + setQueriedTotal(message.data.total) + break; + case "startTime": + setStartTime(message.data); + break; default: console.error(`unsupported websocket message type, Got: ${message.messageType}`) } @@ -111,7 +154,7 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS useEffect(() => { (async () => { - openWebSocket(); + openWebSocket("rlimit(100)"); try{ const tapStatusResponse = await api.tapStatus(); setTappingStatus(tapStatusResponse); @@ -139,14 +182,17 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS }, [focusedEntryId]) const toggleConnection = () => { - setConnection(connection === ConnectionStatus.Connected ? ConnectionStatus.Paused : ConnectionStatus.Connected); + if (connection === ConnectionStatus.Connected) { + ws.current.close(); + } else { + openWebSocket(query); + setConnection(ConnectionStatus.Connected); + } } const getConnectionStatusClass = (isContainer) => { const container = isContainer ? "Container" : ""; switch (connection) { - case ConnectionStatus.Paused: - return "orangeIndicator" + container; case ConnectionStatus.Connected: return "greenIndicator" + container; default: @@ -156,8 +202,6 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS const getConnectionTitle = () => { switch (connection) { - case ConnectionStatus.Paused: - return "traffic paused"; case ConnectionStatus.Connected: return "connected, waiting for traffic" default: @@ -176,8 +220,10 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS return (
- {connection !== ConnectionStatus.Closed && pause} + pause + play
{getConnectionTitle()}
@@ -185,42 +231,51 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS
- {entries.length > 0 &&
+ {
-
-
- {selectedEntryData && } + {selectedEntryData && }
} {tappingStatus?.pods != null && } +
) }; diff --git a/ui/src/components/UI/EndpointPath.tsx b/ui/src/components/UI/EndpointPath.tsx deleted file mode 100644 index 2561aab44..000000000 --- a/ui/src/components/UI/EndpointPath.tsx +++ /dev/null @@ -1,15 +0,0 @@ -import miscStyles from "./style/misc.module.sass"; -import React from "react"; -import styles from './style/EndpointPath.module.sass'; - -interface EndpointPathProps { - method: string, - path: string -} - -export const EndpointPath: React.FC = ({method, path}) => { - return
- {method && {method}} - {path &&
{path}
} -
-}; diff --git a/ui/src/components/UI/FilterSelect.tsx b/ui/src/components/UI/FilterSelect.tsx deleted file mode 100644 index bf6764ad0..000000000 --- a/ui/src/components/UI/FilterSelect.tsx +++ /dev/null @@ -1,28 +0,0 @@ -import React from "react"; -import { MenuItem } from '@material-ui/core'; -import style from './style/FilterSelect.module.sass'; -import { Select, SelectProps } from "./Select"; - -interface FilterSelectProps extends SelectProps { - items: string[]; - value: string | string[]; - onChange: (string) => void; - label?: string; - allowMultiple?: boolean; - transformDisplay?: (string) => string; -} - -export const FilterSelect: React.FC = ({items, value, onChange, label, allowMultiple= false, transformDisplay}) => { - return -}; diff --git a/ui/src/components/UI/Protocol.tsx b/ui/src/components/UI/Protocol.tsx index 0b031bf18..0befffce0 100644 --- a/ui/src/components/UI/Protocol.tsx +++ b/ui/src/components/UI/Protocol.tsx @@ -4,7 +4,8 @@ import styles from './style/Protocol.module.sass'; export interface ProtocolInterface { name: string longName: string - abbreviation: string + abbr: string + macro: string backgroundColor: string foregroundColor: string fontSize: number @@ -16,9 +17,10 @@ export interface ProtocolInterface { interface ProtocolProps { protocol: ProtocolInterface horizontal: boolean + updateQuery: any } -const Protocol: React.FC = ({protocol, horizontal}) => { +const Protocol: React.FC = ({protocol, horizontal, updateQuery}) => { if (horizontal) { return = ({protocol, horizontal}) => { color: protocol.foregroundColor, fontSize: 13, }} - title={protocol.abbreviation} + title={protocol.abbr} > {protocol.longName} } else { - return - - {protocol.abbreviation} - - + return { + updateQuery(protocol.macro) + }} + > + {protocol.abbr} + } }; diff --git a/ui/src/components/UI/StatusCode.tsx b/ui/src/components/UI/StatusCode.tsx index b35788ac1..b743aec02 100644 --- a/ui/src/components/UI/StatusCode.tsx +++ b/ui/src/components/UI/StatusCode.tsx @@ -9,16 +9,21 @@ export enum StatusCodeClassification { interface EntryProps { statusCode: number + updateQuery: any } -const StatusCode: React.FC = ({statusCode}) => { +const StatusCode: React.FC = ({statusCode, updateQuery}) => { const classification = getClassification(statusCode) return - {statusCode} + className={`queryable ${styles[classification]} ${styles.base}`} + onClick={() => { + updateQuery(`response.status == ${statusCode}`) + }} + > + {statusCode} }; diff --git a/ui/src/components/UI/Summary.tsx b/ui/src/components/UI/Summary.tsx new file mode 100644 index 000000000..3c05a89a8 --- /dev/null +++ b/ui/src/components/UI/Summary.tsx @@ -0,0 +1,32 @@ +import miscStyles from "./style/misc.module.sass"; +import React from "react"; +import styles from './style/Summary.module.sass'; + +interface SummaryProps { + method: string + summary: string + updateQuery: any +} + +export const Summary: React.FC = ({method, summary, updateQuery}) => { + return
+ {method && { + updateQuery(`method == "${method}"`) + }} + > + {method} + } + {summary &&
{ + updateQuery(`summary == "${summary}"`) + }} + > + {summary} +
} +
+}; diff --git a/ui/src/components/UI/SyntaxHighlighter/highlighterStyle.ts b/ui/src/components/UI/SyntaxHighlighter/highlighterStyle.ts index a5be67b25..1766e309a 100644 --- a/ui/src/components/UI/SyntaxHighlighter/highlighterStyle.ts +++ b/ui/src/components/UI/SyntaxHighlighter/highlighterStyle.ts @@ -112,7 +112,7 @@ export const highlighterStyle = { "color": "#C6C5FE" }, "operator": { - "color": "#EDEDED" + "color": "#A1A1A1" }, "entity": { "color": "#fdab2b", diff --git a/ui/src/components/UI/style/Protocol.module.sass b/ui/src/components/UI/style/Protocol.module.sass index e702f5ae2..9ffeaab14 100644 --- a/ui/src/components/UI/style/Protocol.module.sass +++ b/ui/src/components/UI/style/Protocol.module.sass @@ -6,7 +6,6 @@ background-color: #000 color: #fff margin-left: -8px - margin-bottom: -4px .vertical line-height: 22px diff --git a/ui/src/components/UI/style/EndpointPath.module.sass b/ui/src/components/UI/style/Summary.module.sass similarity index 75% rename from ui/src/components/UI/style/EndpointPath.module.sass rename to ui/src/components/UI/style/Summary.module.sass index 2fc54c0f3..59bc34893 100644 --- a/ui/src/components/UI/style/EndpointPath.module.sass +++ b/ui/src/components/UI/style/Summary.module.sass @@ -2,7 +2,7 @@ display: flex align-items: center -.path +.summary text-overflow: ellipsis overflow: hidden - white-space: nowrap \ No newline at end of file + white-space: nowrap diff --git a/ui/src/components/assets/filter-ui-example-1.png b/ui/src/components/assets/filter-ui-example-1.png new file mode 100644 index 000000000..244c1b511 Binary files /dev/null and b/ui/src/components/assets/filter-ui-example-1.png differ diff --git a/ui/src/components/assets/filter-ui-example-2.png b/ui/src/components/assets/filter-ui-example-2.png new file mode 100644 index 000000000..97424b2a5 Binary files /dev/null and b/ui/src/components/assets/filter-ui-example-2.png differ diff --git a/ui/src/components/style/EntriesList.module.sass b/ui/src/components/style/EntriesList.module.sass index 453f46081..c787799ab 100644 --- a/ui/src/components/style/EntriesList.module.sass +++ b/ui/src/components/style/EntriesList.module.sass @@ -38,15 +38,6 @@ border: 1px solid #627ef7 background-color: rgba(255, 255, 255, 0.06) -.spinnerContainer - display: flex - justify-content: center - margin-bottom: 10px - -.noMoreDataAvailable - text-align: center - font-weight: 600 - color: $secondary-font-color .fetchButtonContainer width: 100% display: flex diff --git a/ui/src/components/style/Filters.module.sass b/ui/src/components/style/Filters.module.sass index ae8af8224..b58795bad 100644 --- a/ui/src/components/style/Filters.module.sass +++ b/ui/src/components/style/Filters.module.sass @@ -4,9 +4,6 @@ display: flex flex-direction: row align-items: center - min-height: 3rem - overflow-y: hidden - overflow-x: auto padding: .5rem 0 border-bottom: 1px solid #BCC6DD margin-right: 20px @@ -29,8 +26,8 @@ input padding: 4px 12px background: $main-background-color - border-radius: 12px - font-size: 12px + border-radius: 4px + font-size: 14px border: 1px solid #BCC6DD fieldset border: none diff --git a/ui/src/components/style/TrafficPage.sass b/ui/src/components/style/TrafficPage.sass index ceeefe128..3a3c558f5 100644 --- a/ui/src/components/style/TrafficPage.sass +++ b/ui/src/components/style/TrafficPage.sass @@ -110,3 +110,8 @@ align-items: center height: 17px font-size: 16px + +.playPauseIcon + cursor: pointer + margin-right: 15px + height: 30px \ No newline at end of file diff --git a/ui/src/helpers/api.js b/ui/src/helpers/api.js index 3e6d1de65..2c92eb279 100644 --- a/ui/src/helpers/api.js +++ b/ui/src/helpers/api.js @@ -29,13 +29,8 @@ export default class Api { return response.data; } - getEntry = async (entryId) => { - const response = await this.client.get(`/entries/${entryId}`); - return response.data; - } - - fetchEntries = async (operator, timestamp) => { - const response = await this.client.get(`/entries?limit=50&operator=${operator}×tamp=${timestamp}`); + getEntry = async (id) => { + const response = await this.client.get(`/entries/${id}`); return response.data; } @@ -48,4 +43,11 @@ export default class Api { const response = await this.client.get("/status/auth"); return response.data; } + + validateQuery = async (query) => { + const form = new FormData(); + form.append('query', query) + const response = await this.client.post(`/query/validate`, form); + return response.data; + } } diff --git a/ui/src/index.sass b/ui/src/index.sass index 2cb79f24b..256f72501 100644 --- a/ui/src/index.sass +++ b/ui/src/index.sass @@ -22,6 +22,11 @@ code .uppercase text-transform: uppercase +.queryable + cursor: pointer + &:hover + text-decoration: underline + /**** * Button ***/ @@ -31,11 +36,13 @@ button &:not(.MuiFab-root) &.MuiButtonBase-root box-sizing: border-box - font-weight: 500 + font-weight: 600 line-height: 1 - border-radius: 20px + border-radius: 4px letter-spacing: 0.02857em - text-transform: uppercase + background-color: $blue-color + color: #fff + text-transform: none img:not(.custom) max-width: 13px max-height: 13px