diff --git a/Dockerfile b/Dockerfile index e442cd60f..e85745689 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM node:14-slim AS site-build -WORKDIR /ui-build +WORKDIR /app/ui-build COPY ui . RUN npm i @@ -14,14 +14,16 @@ ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64 RUN apk add libpcap-dev gcc g++ make # Move to api working directory (/api-build). -WORKDIR /api-build +WORKDIR /app/api-build COPY api/go.mod api/go.sum ./ +COPY shared/go.mod shared/go.mod ../shared/ RUN go mod download # cheap trick to make the build faster (As long as go.mod wasn't changes) RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get # Copy and build api code +COPY shared ../shared COPY api . RUN go build -ldflags="-s -w" -o mizuagent . @@ -32,10 +34,10 @@ RUN apk add bash libpcap-dev tcpdump WORKDIR /app # Copy binary and config files from /build to root folder of scratch container. -COPY --from=builder ["/api-build/mizuagent", "."] -COPY --from=site-build ["/ui-build/build", "site"] +COPY --from=builder ["/app/api-build/mizuagent", "."] +COPY --from=site-build ["/app/ui-build/build", "site"] COPY api/start.sh . # this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process -CMD "./start.sh" +ENTRYPOINT "/app/mizuagent" diff --git a/Makefile b/Makefile index eb481cc9d..b304676bc 100644 --- a/Makefile +++ b/Makefile @@ -38,8 +38,7 @@ api: ## build API server docker: ## build Docker image @(echo "building docker image" ) - docker build -t up9inc/mizu:latest . - #./build-push-featurebranch.sh + ./build-push-featurebranch.sh push: push-docker push-cli ## build and publish Mizu docker image & CLI diff --git a/api/go.mod b/api/go.mod index 3069f5302..4f4c055c0 100644 --- a/api/go.mod +++ b/api/go.mod @@ -3,37 +3,8 @@ module mizuserver go 1.16 require ( - cloud.google.com/go v0.54.0 // indirect - cloud.google.com/go/bigquery v1.4.0 // indirect - cloud.google.com/go/datastore v1.1.0 // indirect - cloud.google.com/go/pubsub v1.2.0 // indirect - cloud.google.com/go/storage v1.6.0 // indirect - github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.12 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.5 // indirect - github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect - github.com/Azure/go-autorest/autorest/mocks v0.4.1 // indirect - github.com/Azure/go-autorest/logger v0.2.0 // indirect - github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/BurntSushi/toml v0.3.1 // indirect - github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 // indirect - github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 // indirect - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/andybalholm/brotli v1.0.1 // indirect github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a - github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect - github.com/aws/aws-sdk-go v1.34.28 // indirect - github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect - github.com/chzyer/logex v1.1.10 // indirect - github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect - github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 // indirect - github.com/client9/misspell v0.3.4 // indirect - github.com/creack/pty v1.1.9 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/djherbis/atime v1.0.0 - github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 // indirect - github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 // indirect github.com/fasthttp/websocket v1.4.3-beta.1 // indirect github.com/go-playground/locales v0.13.0 github.com/go-playground/universal-translator v0.17.0 @@ -45,12 +16,14 @@ require ( github.com/leodido/go-urn v1.2.1 // indirect github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 github.com/patrickmn/go-cache v2.1.0+incompatible - github.com/stretchr/objx v0.2.0 // indirect + github.com/up9inc/mizu/shared v0.0.0 go.mongodb.org/mongo-driver v1.5.1 golang.org/x/net v0.0.0-20210421230115-4e50805a0758 gorm.io/driver/sqlite v1.1.4 gorm.io/gorm v1.21.8 - k8s.io/api v0.21.0 // indirect + k8s.io/api v0.21.0 k8s.io/apimachinery v0.21.0 k8s.io/client-go v0.21.0 ) + +replace github.com/up9inc/mizu/shared v0.0.0 => ../shared diff --git a/api/go.sum b/api/go.sum index 2a30e989f..bc655181d 100644 --- a/api/go.sum +++ b/api/go.sum @@ -12,20 +12,15 @@ cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -41,79 +36,53 @@ github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8 github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 h1:lsxEuwrXEAokXB9qhlbKWPpo3KMLZQ5WB5WLQRW1uq0= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a h1:76llBleIE3fkdqaJFDzdirtiYhQPdIQem8H8r2iwA1Q= github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a/go.mod h1:QvDfsDQDmGxUsvEeWabVZ5pp2FMXpOkwQV0L6SE6cp0= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= -github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg= github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473 h1:4cmBvAEBNJaGARUEs3/suWRyfyBfhf7I60WBZq+bv2w= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fasthttp/websocket v1.4.2/go.mod h1:smsv/h4PBEBaU0XDTY5UwJTpZv69fQ0FfcLJr21mA6Y= github.com/fasthttp/websocket v1.4.3-beta.1 h1:stc4P2aoxYKsdmbe1AJ5mAm73Fxc1NOgrZpPftvZIXQ= github.com/fasthttp/websocket v1.4.3-beta.1/go.mod h1:JGrgLaT02bL9NuJkZbHN8mVV2tkCJZQh7yJ5/XCXO2g= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -123,45 +92,31 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E= github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk= -github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd h1:hSkbZ9XSyjyBirMeqSqUrK+9HboWrweVlzRNqoBi2d4= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0 h1:31atYa/UW9V5q8vMJ+W6wd64OaaTHUrCUXER358zLM4= github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3 h1:3GQ53z7E3o00C/yy7Ko8VXqQXoJGLkrTQCLTF1EjoXU= github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1 h1:iQ0D6SpNXIxu52WESsD+KoQ7af2e3nCfnSBoSF/hKe0= github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211 h1:mSVZ4vj4khv+oThUfS+SQU3UuFIZ5Zo6UNcvK8E8Mz8= github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1 h1:dLg+zb+uOyd/mKeQUYIbwbNmfRsr9hd/WtYWepmayhI= github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2 h1:8thhT+kUJMTMy3HlX4+y9Da+BNJck+p109tqqKp7WDs= github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2 h1:fq9WcL1BYrm36SzK6+aAnZ8hcp+SrmnDyAxhNx8dvJk= github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0 h1:4sGKOD8yaYJ+dek1FDkwcxCHA40M4kfKgFHx8N2kwbU= github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0 h1:Ir9W9XIm9j7bhhkKE9cokvtTl1vBm62A/fene/ZCj6A= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754 h1:tpom+2CJmpzAWj5/VEHync2rJGi+epHNIeRSWjzGA+4= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gofiber/fiber/v2 v2.1.3/go.mod h1:MMiSv1HrDkN8Pv7NeVDYK+T/lwXOEKAvPBbLvJPCEfA= github.com/gofiber/fiber/v2 v2.7.1/go.mod h1:f8BRRIMjMdRyt2qmJ/0Sea3j3rwwfufPrh9WNBRiVZ0= @@ -171,17 +126,14 @@ github.com/gofiber/websocket/v2 v2.0.3 h1:nqPGHB4LQhxKX5KJUjayOd2xiiENieS/dn6TPf github.com/gofiber/websocket/v2 v2.0.3/go.mod h1:/OTEImCxORKE5unw0dWqJYovid6vZF+wB1W0aaMKs2M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1 h1:ocYkMQY5RrXTYgXl7ICpV0IXwlEQGwKIsery4gyXa1U= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -196,10 +148,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -219,56 +169,40 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3 h1:SRgJV+IoxM5MKyFdlSUeNy6/ycRUF2yBAKdAQswoHUk= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3 h1:lOpSw2vJP0y5eLBW906QwKsUK/fe/QDyoqM5rnnuPDY= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -276,20 +210,13 @@ github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5 h1:hyz3dwM5QLc1Rfoz4FuWJQG5BN7tc6K1MndAUnGpQr4= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -297,17 +224,12 @@ github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgx github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2 h1:JgVTCPf0uBVcUSWpyXmGpgOc62nK5HWUBKAGc3Qqa5k= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ= github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -315,54 +237,40 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d h1:7PxY7LVfSZm7PEeBTyK1rj1gABdCO2mbri6GKO1cMDs= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU= github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY= github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c h1:KKqhycXW1WVNkX7r4ekTV2gFkbhdyihlWD8c0/FiWmk= github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -376,7 +284,6 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= @@ -388,23 +295,17 @@ github.com/valyala/fasthttp v1.23.0 h1:0ufwSD9BhWa6f8HWdmdq4FHQ23peRo3Ng/Qs8m5Nc github.com/valyala/fasthttp v1.23.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.mongodb.org/mongo-driver v1.5.1 h1:9nOVLGDfOaZ9R0tBumx/BcuqkbFpyTCU2r/Po7A2azI= go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -427,10 +328,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -441,18 +340,14 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee h1:WG0RUwxtNT4qqaXX3DPA8zHFNm/D9xaBpxzHt1WcA/E= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -494,9 +389,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -534,7 +427,6 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe h1:WdX7u8s3yOigWAhHEaDl8r9G+4XwFQEQFtBMYyN+kXQ= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -581,7 +473,6 @@ golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7 h1:EBZoQjiKKPaLbPrbpssUfuHtwM6KV/vb4U85g/cigFY= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -589,11 +480,9 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -606,7 +495,6 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -632,7 +520,6 @@ google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -640,7 +527,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -653,22 +539,17 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -684,7 +565,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y= k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU= @@ -692,20 +572,15 @@ k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA= k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= k8s.io/client-go v0.21.0 h1:n0zzzJsAQmJngpC0IhgFcApZyoGXPrDIAD601HD09ag= k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac h1:sAvhNk5RRuc6FNYGqe7Ygz3PSo/2wGWbulskmzRX8Vs= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= diff --git a/api/main.go b/api/main.go index e8d4242f1..220b53a69 100644 --- a/api/main.go +++ b/api/main.go @@ -1,24 +1,74 @@ package main import ( + "encoding/json" "flag" + "fmt" "github.com/gofiber/fiber/v2" - "mizuserver/pkg/inserter" + "github.com/gorilla/websocket" + "github.com/up9inc/mizu/shared" + "mizuserver/pkg/api" "mizuserver/pkg/middleware" + "mizuserver/pkg/models" "mizuserver/pkg/routes" "mizuserver/pkg/tap" "mizuserver/pkg/utils" + "os" + "os/signal" ) +var shouldTap = flag.Bool("tap", false, "Run in tapper mode without API") +var aggregator = flag.Bool("aggregator", false, "Run in aggregator mode with API") +var standalone = flag.Bool("standalone", false, "Run in standalone tapper and API mode") +var aggregatorAddress = flag.String("aggregator-address", "", "Address of mizu collector for tapping") + +const nodeNameEnvVar = "NODE_NAME" +const tappedAddressesPerNodeDictEnvVar = "TAPPED_ADDRESSES_PER_HOST" + func main() { flag.Parse() - harOutputChannel := tap.StartPassiveTapper() + if !*shouldTap && !*aggregator && !*standalone{ + panic("One of the flags --tap, --api or --standalone must be provided") + } + if *standalone { + harOutputChannel := tap.StartPassiveTapper() + go api.StartReadingEntries(harOutputChannel, tap.HarOutputDir) + hostApi(nil) + } else if *shouldTap { + if *aggregatorAddress == "" { + panic("Aggregator address must be provided with --aggregator-address when using --tap") + } + + tapTargets := getTapTargets() + if tapTargets != nil { + tap.HostAppAddresses = tapTargets + fmt.Println("Filtering for the following addresses:", tap.HostAppAddresses) + } + + harOutputChannel := tap.StartPassiveTapper() + socketConnection, err := shared.ConnectToSocketServer(*aggregatorAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false) + if err != nil { + panic(fmt.Sprintf("Error connecting to socket server at %s %v", *aggregatorAddress, err)) + } + go pipeChannelToSocket(socketConnection, harOutputChannel) + } else if *aggregator { + socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000) + go api.StartReadingEntries(socketHarOutChannel, nil) + hostApi(socketHarOutChannel) + } + + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, os.Interrupt) + <-signalChan + + fmt.Println("Exiting") +} + +func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) { app := fiber.New() - // process to read files / channel and insert to DB - go inserter.StartReadingEntries(harOutputChannel, tap.HarOutputDir) middleware.FiberMiddleware(app) // Register Fiber's middleware for app. app.Static("/", "./site") @@ -27,10 +77,47 @@ func main() { app.Get("/echo", func(c *fiber.Ctx) error { return c.SendString("Hello, World 👋!") }) - - routes.WebSocketRoutes(app) + eventHandlers := api.RoutesEventHandlers{ + SocketHarOutChannel: socketHarOutputChannel, + } + routes.WebSocketRoutes(app, &eventHandlers) routes.EntriesRoutes(app) routes.NotFoundRoute(app) utils.StartServer(app) } + + +func getTapTargets() []string { + nodeName := os.Getenv(nodeNameEnvVar) + var tappedAddressesPerNodeDict map[string][]string + err := json.Unmarshal([]byte(os.Getenv(tappedAddressesPerNodeDictEnvVar)), &tappedAddressesPerNodeDict) + if err != nil { + panic(fmt.Sprintf("env var value of %s is invalid! must be map[string][]string %v", tappedAddressesPerNodeDict, err)) + } + return tappedAddressesPerNodeDict[nodeName] +} + +func pipeChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) { + if connection == nil { + panic("Websocket connection is nil") + } + + if messageDataChannel == nil { + panic("Channel of captured messages is nil") + } + + for messageData := range messageDataChannel { + marshaledData, err := models.CreateWebsocketTappedEntryMessage(messageData) + if err != nil { + fmt.Printf("error converting message to json %s, (%v,%+v)\n", err, err, err) + continue + } + + err = connection.WriteMessage(websocket.TextMessage, marshaledData) + if err != nil { + fmt.Printf("error sending message through socket server %s, (%v,%+v)\n", err, err, err) + continue + } + } +} diff --git a/api/pkg/inserter/main.go b/api/pkg/api/main.go similarity index 73% rename from api/pkg/inserter/main.go rename to api/pkg/api/main.go index 4089fb01d..1aea81795 100644 --- a/api/pkg/inserter/main.go +++ b/api/pkg/api/main.go @@ -1,11 +1,10 @@ -package inserter +package api import ( "bufio" "context" "encoding/json" "fmt" - "github.com/antoniodipinto/ikisocket" "github.com/google/martian/har" "go.mongodb.org/mongo-driver/bson/primitive" "mizuserver/pkg/database" @@ -34,7 +33,7 @@ func init() { go func() { for { select { - case err := <- errOut: + case err := <-errOut: fmt.Printf("name resolving error %s", err) } } @@ -43,7 +42,7 @@ func init() { k8sResolver = res } -func StartReadingEntries(harChannel chan *tap.OutputChannelItem, workingDir *string) { +func StartReadingEntries(harChannel <-chan *tap.OutputChannelItem, workingDir *string) { if workingDir != nil && *workingDir != "" { startReadingFiles(*workingDir) } else { @@ -83,7 +82,11 @@ func startReadingFiles(workingDir string) { } } -func startReadingChannel(outputItems chan *tap.OutputChannelItem) { +func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) { + if outputItems == nil { + panic("Channel of captured messages is nil") + } + for item := range outputItems { saveHarToDb(item.HarEntry, item.RequestSenderIp) } @@ -94,7 +97,7 @@ func saveHarToDb(entry *har.Entry, sender string) { serviceName, urlPath, serviceHostName := getServiceNameFromUrl(entry.Request.URL) entryId := primitive.NewObjectID().Hex() var ( - resolvedSource *string + resolvedSource *string resolvedDestination *string ) if k8sResolver != nil { @@ -102,23 +105,23 @@ func saveHarToDb(entry *har.Entry, sender string) { resolvedDestination = k8sResolver.Resolve(serviceHostName) } mizuEntry := models.MizuEntry{ - EntryId: entryId, - Entry: string(entryBytes), // simple way to store it and not convert to bytes - Service: serviceName, - Url: entry.Request.URL, - Path: urlPath, - Method: entry.Request.Method, - Status: entry.Response.Status, - RequestSenderIp: sender, - Timestamp: entry.StartedDateTime.UnixNano() / int64(time.Millisecond), - ResolvedSource: resolvedSource, + EntryId: entryId, + Entry: string(entryBytes), // simple way to store it and not convert to bytes + Service: serviceName, + Url: entry.Request.URL, + Path: urlPath, + Method: entry.Request.Method, + Status: entry.Response.Status, + RequestSenderIp: sender, + Timestamp: entry.StartedDateTime.UnixNano() / int64(time.Millisecond), + ResolvedSource: resolvedSource, ResolvedDestination: resolvedDestination, } database.GetEntriesTable().Create(&mizuEntry) baseEntry := utils.GetResolvedBaseEntry(mizuEntry) - baseEntryBytes, _ := json.Marshal(&baseEntry) - ikisocket.Broadcast(baseEntryBytes) + baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry) + broadcastToBrowserClients(baseEntryBytes) } func getServiceNameFromUrl(inputUrl string) (string, string, string) { @@ -126,4 +129,3 @@ func getServiceNameFromUrl(inputUrl string) (string, string, string) { utils.CheckErr(err) return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host), parsed.Path, parsed.Host } - diff --git a/api/pkg/api/socket_server_handlers.go b/api/pkg/api/socket_server_handlers.go new file mode 100644 index 000000000..12bf42adf --- /dev/null +++ b/api/pkg/api/socket_server_handlers.go @@ -0,0 +1,96 @@ +package api + +import ( + "encoding/json" + "fmt" + "github.com/antoniodipinto/ikisocket" + "github.com/up9inc/mizu/shared" + "mizuserver/pkg/controllers" + "mizuserver/pkg/models" + "mizuserver/pkg/routes" + "mizuserver/pkg/tap" +) + +var browserClientSocketUUIDs = make([]string, 0) + +type RoutesEventHandlers struct { + routes.EventHandlers + SocketHarOutChannel chan<- *tap.OutputChannelItem +} + + +func (h *RoutesEventHandlers) WebSocketConnect(ep *ikisocket.EventPayload) { + if ep.Kws.GetAttribute("is_tapper") == true { + fmt.Println(fmt.Sprintf("Websocket Connection event - Tapper connected: %s", ep.SocketUUID)) + } else { + fmt.Println(fmt.Sprintf("Websocket Connection event - Browser socket connected: %s", ep.SocketUUID)) + browserClientSocketUUIDs = append(browserClientSocketUUIDs, ep.SocketUUID) + } +} + +func (h *RoutesEventHandlers) WebSocketDisconnect(ep *ikisocket.EventPayload) { + if ep.Kws.GetAttribute("is_tapper") == true { + fmt.Println(fmt.Sprintf("Disconnection event - Tapper connected: %s", ep.SocketUUID)) + } else { + fmt.Println(fmt.Sprintf("Disconnection event - Browser socket connected: %s", ep.SocketUUID)) + removeSocketUUIDFromBrowserSlice(ep.SocketUUID) + } +} + +func broadcastToBrowserClients(message []byte) { + ikisocket.EmitToList(browserClientSocketUUIDs, message) +} + +func (h *RoutesEventHandlers) WebSocketClose(ep *ikisocket.EventPayload) { + if ep.Kws.GetAttribute("is_tapper") == true { + fmt.Println(fmt.Sprintf("Websocket Close event - Tapper connected: %s", ep.SocketUUID)) + } else { + fmt.Println(fmt.Sprintf("Websocket Close event - Browser socket connected: %s", ep.SocketUUID)) + removeSocketUUIDFromBrowserSlice(ep.SocketUUID) + } +} + +func (h *RoutesEventHandlers) WebSocketError(ep *ikisocket.EventPayload) { + fmt.Println(fmt.Sprintf("Socket error - Socket uuid : %s %v", ep.SocketUUID, ep.Error)) +} + +func (h *RoutesEventHandlers) WebSocketMessage(ep *ikisocket.EventPayload) { + var socketMessageBase shared.WebSocketMessageMetadata + err := json.Unmarshal(ep.Data, &socketMessageBase) + if err != nil { + fmt.Printf("Could not unmarshal websocket message %v\n", err) + } else { + switch socketMessageBase.MessageType { + case shared.WebSocketMessageTypeTappedEntry: + var tappedEntryMessage models.WebSocketTappedEntryMessage + err := json.Unmarshal(ep.Data, &tappedEntryMessage) + if err != nil { + fmt.Printf("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err) + } else { + h.SocketHarOutChannel <- tappedEntryMessage.Data + } + case shared.WebSocketMessageTypeUpdateStatus: + var statusMessage shared.WebSocketStatusMessage + err := json.Unmarshal(ep.Data, &statusMessage) + if err != nil { + fmt.Printf("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err) + } else { + controllers.TapStatus = statusMessage.TappingStatus + broadcastToBrowserClients(ep.Data) + } + default: + fmt.Printf("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType) + } + } +} + + +func removeSocketUUIDFromBrowserSlice(uuidToRemove string) { + newUUIDSlice := make([]string, 0, len(browserClientSocketUUIDs)) + for _, uuid := range browserClientSocketUUIDs { + if uuid != uuidToRemove { + newUUIDSlice = append(newUUIDSlice, uuid) + } + } + browserClientSocketUUIDs = newUUIDSlice +} diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index 46977b6b1..e9702fe3b 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -64,6 +64,65 @@ func GetEntries(c *fiber.Ctx) error { return c.Status(fiber.StatusOK).JSON(baseEntries) } +func GetHAR(c *fiber.Ctx) error { + entriesFilter := &models.HarFetchRequestBody{} + order := OrderDesc + if err := c.QueryParser(entriesFilter); err != nil { + return c.Status(fiber.StatusBadRequest).JSON(err) + } + err := validation.Validate(entriesFilter) + if err != nil { + return c.Status(fiber.StatusBadRequest).JSON(err) + } + + var entries []models.MizuEntry + database.GetEntriesTable(). + Order(fmt.Sprintf("timestamp %s", order)). + // Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)). + Limit(1000). + Find(&entries) + + if len(entries) > 0 { + // the entries always order from oldest to newest so we should revers + utils.ReverseSlice(entries) + } + + harsObject := map[string]*har.HAR{} + + for _, entryData := range entries { + harEntryObject := []byte(entryData.Entry) + + var harEntry har.Entry + _ = json.Unmarshal(harEntryObject, &harEntry) + + sourceOfEntry := *entryData.ResolvedSource + if harOfSource, ok := harsObject[sourceOfEntry]; ok { + harOfSource.Log.Entries = append(harOfSource.Log.Entries, &harEntry) + } else { + var entriesHar []*har.Entry + entriesHar = append(entriesHar, &harEntry) + harsObject[sourceOfEntry] = &har.HAR{ + Log: &har.Log{ + Version: "1.2", + Creator: &har.Creator{ + Name: "mizu", + Version: "0.0.1", + }, + Entries: entriesHar, + }, + } + } + } + + retObj := map[string][]byte{} + for k, v := range harsObject { + bytesData, _ := json.Marshal(v) + retObj[k] = bytesData + } + buffer := utils.ZipData(retObj) + return c.Status(fiber.StatusOK).SendStream(buffer) +} + func GetEntry(c *fiber.Ctx) error { var entryData models.EntryData database.GetEntriesTable(). diff --git a/api/pkg/controllers/status_controller.go b/api/pkg/controllers/status_controller.go new file mode 100644 index 000000000..15f30a67b --- /dev/null +++ b/api/pkg/controllers/status_controller.go @@ -0,0 +1,12 @@ +package controllers + +import ( + "github.com/gofiber/fiber/v2" + "github.com/up9inc/mizu/shared" +) + +var TapStatus shared.TapStatus + +func GetTappingStatus(c *fiber.Ctx) error { + return c.Status(fiber.StatusOK).JSON(TapStatus) +} diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index d85387d40..feb1a4b90 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -1,21 +1,26 @@ package models -import "time" +import ( + "encoding/json" + "github.com/up9inc/mizu/shared" + "mizuserver/pkg/tap" + "time" +) type MizuEntry struct { - ID uint `gorm:"primarykey"` - CreatedAt time.Time - UpdatedAt time.Time - Entry string `json:"entry,omitempty" gorm:"column:entry"` - EntryId string `json:"entryId" gorm:"column:entryId"` - Url string `json:"url" gorm:"column:url"` - Method string `json:"method" gorm:"column:method"` - Status int `json:"status" gorm:"column:status"` - RequestSenderIp string `json:"requestSenderIp" gorm:"column:requestSenderIp"` - Service string `json:"service" gorm:"column:service"` - Timestamp int64 `json:"timestamp" gorm:"column:timestamp"` - Path string `json:"path" gorm:"column:path"` - ResolvedSource *string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"` + ID uint `gorm:"primarykey"` + CreatedAt time.Time + UpdatedAt time.Time + Entry string `json:"entry,omitempty" gorm:"column:entry"` + EntryId string `json:"entryId" gorm:"column:entryId"` + Url string `json:"url" gorm:"column:url"` + Method string `json:"method" gorm:"column:method"` + Status int `json:"status" gorm:"column:status"` + RequestSenderIp string `json:"requestSenderIp" gorm:"column:requestSenderIp"` + Service string `json:"service" gorm:"column:service"` + Timestamp int64 `json:"timestamp" gorm:"column:timestamp"` + Path string `json:"path" gorm:"column:path"` + ResolvedSource *string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"` ResolvedDestination *string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` } @@ -31,7 +36,7 @@ type BaseEntryDetails struct { } type EntryData struct { - Entry string `json:"entry,omitempty"` + Entry string `json:"entry,omitempty"` ResolvedDestination *string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` } @@ -40,3 +45,38 @@ type EntriesFilter struct { Operator string `query:"operator" validate:"required,oneof='lt' 'gt'"` Timestamp int64 `query:"timestamp" validate:"required,min=1"` } + +type HarFetchRequestBody struct { + Limit int `query:"limit" validate:"max=5000"` +} + +type WebSocketEntryMessage struct { + *shared.WebSocketMessageMetadata + Data *BaseEntryDetails `json:"data,omitempty"` +} + + +type WebSocketTappedEntryMessage struct { + *shared.WebSocketMessageMetadata + Data *tap.OutputChannelItem +} + +func CreateBaseEntryWebSocketMessage(base *BaseEntryDetails) ([]byte, error) { + message := &WebSocketEntryMessage{ + WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{ + MessageType: shared.WebSocketMessageTypeEntry, + }, + Data: base, + } + return json.Marshal(message) +} + +func CreateWebsocketTappedEntryMessage(base *tap.OutputChannelItem) ([]byte, error) { + message := &WebSocketTappedEntryMessage{ + WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{ + MessageType: shared.WebSocketMessageTypeTappedEntry, + }, + Data: base, + } + return json.Marshal(message) +} diff --git a/api/pkg/routes/public_routes.go b/api/pkg/routes/public_routes.go index eac2d86ef..c15203189 100644 --- a/api/pkg/routes/public_routes.go +++ b/api/pkg/routes/public_routes.go @@ -12,6 +12,9 @@ func EntriesRoutes(fiberApp *fiber.App) { routeGroup.Get("/entries", controllers.GetEntries) // get entries (base/thin entries) routeGroup.Get("/entries/:entryId", controllers.GetEntry) // get single (full) entry + routeGroup.Get("/har", controllers.GetHAR) routeGroup.Get("/resetDB", controllers.DeleteAllEntries) // get single (full) entry routeGroup.Get("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB + + routeGroup.Get("/tapStatus", controllers.GetTappingStatus) // get tapping status } diff --git a/api/pkg/routes/socket_routes.go b/api/pkg/routes/socket_routes.go index c912253ed..a66cc401b 100644 --- a/api/pkg/routes/socket_routes.go +++ b/api/pkg/routes/socket_routes.go @@ -1,43 +1,31 @@ package routes import ( - "fmt" "github.com/antoniodipinto/ikisocket" "github.com/gofiber/fiber/v2" ) -func webSocketConnect(ep *ikisocket.EventPayload) { - fmt.Println(fmt.Sprintf("Connection event 1 - User: %s", ep.Kws.GetStringAttribute("user_id"))) +type EventHandlers interface { + WebSocketConnect(ep *ikisocket.EventPayload) + WebSocketDisconnect(ep *ikisocket.EventPayload) + WebSocketClose(ep *ikisocket.EventPayload) + WebSocketError(ep *ikisocket.EventPayload) + WebSocketMessage(ep *ikisocket.EventPayload) } -func webSocketDisconnect(ep *ikisocket.EventPayload) { - fmt.Println(fmt.Sprintf("Disconnection event - User: %s", ep.Kws.GetStringAttribute("user_id"))) -} - -func webSocketClose(ep *ikisocket.EventPayload) { - fmt.Println(fmt.Sprintf("Close event - User: %s", ep.Kws.GetStringAttribute("user_id"))) -} - -func webSocketError(ep *ikisocket.EventPayload) { - fmt.Println(fmt.Sprintf("Error event - User: %s", ep.Kws.GetStringAttribute("user_id"))) -} - -func webSocketMessage(ep *ikisocket.EventPayload) { - fmt.Println("Web socket message") - // fmt.Println(fmt.Sprintf("Message event - User: %s - Message: %s", ep.Kws.GetStringAttribute("user_id"), string(ep.Data))) -} - -func WebSocketRoutes(app *fiber.App) { - +func WebSocketRoutes(app *fiber.App, eventHandlers EventHandlers) { app.Get("/ws", ikisocket.New(func(kws *ikisocket.Websocket) { - // kws.Broadcast([]byte(fmt.Sprintf("New user connected: %s and UUID: %s", userId, kws.UUID)), true) - // kws.Emit([]byte(fmt.Sprintf("Hello user with UUID: %s", kws.UUID))) - kws.SetAttribute("user_id", kws.UUID) + kws.SetAttribute("is_tapper", false) })) - ikisocket.On(ikisocket.EventMessage, webSocketMessage) - ikisocket.On(ikisocket.EventConnect, webSocketConnect) - ikisocket.On(ikisocket.EventDisconnect, webSocketDisconnect) - ikisocket.On(ikisocket.EventClose, webSocketClose) // This event is called when the server disconnects the user actively with .Close() method - ikisocket.On(ikisocket.EventError, webSocketError) // On error event + app.Get("/wsTapper", ikisocket.New(func(kws *ikisocket.Websocket) { + // Tapper clients are handled differently, they don't need to receive new message broadcasts. + kws.SetAttribute("is_tapper", true) + })) + + ikisocket.On(ikisocket.EventMessage, eventHandlers.WebSocketMessage) + ikisocket.On(ikisocket.EventConnect, eventHandlers.WebSocketConnect) + ikisocket.On(ikisocket.EventDisconnect, eventHandlers.WebSocketDisconnect) + ikisocket.On(ikisocket.EventClose, eventHandlers.WebSocketClose) // This event is called when the server disconnects the user actively with .Close() method + ikisocket.On(ikisocket.EventError, eventHandlers.WebSocketError) // On error event } diff --git a/api/pkg/tap/passive_tapper.go b/api/pkg/tap/passive_tapper.go index 7858d2e94..b5cacc9e8 100644 --- a/api/pkg/tap/passive_tapper.go +++ b/api/pkg/tap/passive_tapper.go @@ -134,7 +134,7 @@ var nErrors uint var appPorts []int // global var ownIps []string //global var hostMode bool //global -var hostAppAddresses []string //global +var HostAppAddresses []string //global /* minOutputLevel: Error will be printed only if outputLevel is above this value * t: key for errorsMap (counting errors) @@ -198,7 +198,7 @@ func (c *Context) GetCaptureInfo() gopacket.CaptureInfo { return c.CaptureInfo } -func StartPassiveTapper() chan *OutputChannelItem { +func StartPassiveTapper() <-chan *OutputChannelItem { var harWriter *HarWriter if *dumpToHar { harWriter = NewHarWriter(*HarOutputDir, *harEntriesPerFile) @@ -240,8 +240,6 @@ func startPassiveTapper(harWriter *HarWriter) { } else { appPorts = parseAppPorts(appPortsStr) } - hostAppAddresses = parseHostAppAddresses(*hostAppAddressesString) - fmt.Println("Filtering for the following addresses:", hostAppAddresses) tapOutputPort := os.Getenv(OutPortEnvVar) if tapOutputPort == "" { fmt.Println("Received empty/no WEB_SOCKET_PORT env var! falling back to port 8080") @@ -275,7 +273,8 @@ func startPassiveTapper(harWriter *HarWriter) { appPorts = *parsedMessage.Ports } else if parsedMessage.MessageType == "setAddresses" { Debug("Got message from collector. Type: %s, IPs: %v\n", parsedMessage.MessageType, parsedMessage.Addresses) - hostAppAddresses = *parsedMessage.Addresses + HostAppAddresses = *parsedMessage.Addresses + Info("Filtering for the following addresses: %s\n", HostAppAddresses) } } else { Error("Collector-Message-Parsing", "Error parsing message from collector: %s (%v,%+v)\n", err, err, err) diff --git a/api/pkg/tap/tcp_stream_factory.go b/api/pkg/tap/tcp_stream_factory.go index 3cb7bfd2d..23bda51bd 100644 --- a/api/pkg/tap/tcp_stream_factory.go +++ b/api/pkg/tap/tcp_stream_factory.go @@ -85,9 +85,9 @@ func (factory *tcpStreamFactory) WaitGoRoutines() { func (factory *tcpStreamFactory) shouldTap(dstIP string, dstPort int) bool { if hostMode { - if inArrayString(hostAppAddresses, fmt.Sprintf("%s:%d", dstIP, dstPort)) == true { + if inArrayString(HostAppAddresses, fmt.Sprintf("%s:%d", dstIP, dstPort)) == true { return true - } else if inArrayString(hostAppAddresses, dstIP) == true { + } else if inArrayString(HostAppAddresses, dstIP) == true { return true } return false diff --git a/api/pkg/utils/utils.go b/api/pkg/utils/utils.go index a25f21a97..9ecb89522 100644 --- a/api/pkg/utils/utils.go +++ b/api/pkg/utils/utils.go @@ -1,6 +1,7 @@ package utils import ( + "encoding/json" "fmt" "github.com/gofiber/fiber/v2" "log" @@ -43,7 +44,6 @@ func ReverseSlice(data interface{}) { } } - func CheckErr(e error) { if e != nil { log.Printf("%v", e) @@ -51,7 +51,6 @@ func CheckErr(e error) { } } - func SetHostname(address, newHostname string) string { replacedUrl, err := url.Parse(address) if err != nil{ @@ -81,3 +80,8 @@ func GetResolvedBaseEntry(entry models.MizuEntry) models.BaseEntryDetails { RequestSenderIp: entry.RequestSenderIp, } } + +func GetBytesFromStruct(v interface{}) []byte{ + a, _ := json.Marshal(v) + return a +} \ No newline at end of file diff --git a/api/pkg/utils/zip.go b/api/pkg/utils/zip.go new file mode 100644 index 000000000..936eb0f3c --- /dev/null +++ b/api/pkg/utils/zip.go @@ -0,0 +1,20 @@ +package utils + +import ( + "archive/zip" + "bytes" +) + +func ZipData(files map[string][]byte) *bytes.Buffer { + // Create a buffer to write our archive to. + buf := new(bytes.Buffer) + // Create a new zip archive. + zipWriter := zip.NewWriter(buf) + defer func() { _ = zipWriter.Close() }() + + for fileName, fileBytes := range files { + zipFile, _ := zipWriter.Create(fileName) + _, _ = zipFile.Write(fileBytes) + } + return buf +} diff --git a/cli/cmd/fetch.go b/cli/cmd/fetch.go index 24ed884db..c4e8fb913 100644 --- a/cli/cmd/fetch.go +++ b/cli/cmd/fetch.go @@ -1,20 +1,28 @@ package cmd import ( - "fmt" - "github.com/spf13/cobra" ) +type MizuFetchOptions struct { + Limit uint16 + Directory string +} + +var mizuFetchOptions = MizuFetchOptions{} + var fetchCmd = &cobra.Command{ Use: "fetch", - Short: "Download recorded traffic", + Short: "Download recorded traffic to files", RunE: func(cmd *cobra.Command, args []string) error { - fmt.Println("Not implemented") + RunMizuFetch(&mizuFetchOptions) return nil }, } func init() { rootCmd.AddCommand(fetchCmd) + + fetchCmd.Flags().Uint16VarP(&mizuFetchOptions.Limit, "limit", "l", 1000, "Provide a custom limit for entries to fetch") + fetchCmd.Flags().StringVarP(&mizuFetchOptions.Directory, "directory", "d", ".", "Provide a custom directory for fetched entries") } diff --git a/cli/cmd/fetchRunner.go b/cli/cmd/fetchRunner.go new file mode 100644 index 000000000..71db8364d --- /dev/null +++ b/cli/cmd/fetchRunner.go @@ -0,0 +1,94 @@ +package cmd + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "strings" +) + +func RunMizuFetch(fetch *MizuFetchOptions) { + resp, err := http.Get(fmt.Sprintf("http://localhost:8899/api/har?limit=%v", fetch.Limit)) + if err != nil { + log.Fatal(err) + } + + defer func() { _ = resp.Body.Close() }() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Fatal(err) + } + + zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) + if err != nil { + log.Fatal(err) + } + _ = Unzip(zipReader, fetch.Directory) + +} + +func Unzip(reader *zip.Reader, dest string) error { + dest, _ = filepath.Abs(dest) + _ = os.MkdirAll(dest, os.ModePerm) + + // Closure to address file descriptors issue with all the deferred .Close() methods + extractAndWriteFile := func(f *zip.File) error { + rc, err := f.Open() + if err != nil { + return err + } + defer func() { + if err := rc.Close(); err != nil { + panic(err) + } + }() + + path := filepath.Join(dest, f.Name) + + // Check for ZipSlip (Directory traversal) + if !strings.HasPrefix(path, filepath.Clean(dest) + string(os.PathSeparator)) { + return fmt.Errorf("illegal file path: %s", path) + } + + if f.FileInfo().IsDir() { + _ = os.MkdirAll(path, f.Mode()) + } else { + _ = os.MkdirAll(filepath.Dir(path), f.Mode()) + fmt.Print("writing HAR file [ ", path, " ] .. ") + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return err + } + defer func() { + if err := f.Close(); err != nil { + panic(err) + } + fmt.Println(" done") + }() + + _, err = io.Copy(f, rc) + if err != nil { + return err + } + } + return nil + } + + for _, f := range reader.File { + err := extractAndWriteFile(f) + if err != nil { + return err + } + } + + return nil +} + + diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 153983d7d..239d76a62 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -8,7 +8,7 @@ var rootCmd = &cobra.Command{ Use: "mizu", Short: "A web traffic viewer for kubernetes", Long: `A web traffic viewer for kubernetes - Further info is available at https://github.com/up9inc/mizu`, +Further info is available at https://github.com/up9inc/mizu`, } // Execute adds all child commands to the root command and sets flags appropriately. diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 77570d254..c392f99ba 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -3,27 +3,41 @@ package cmd import ( "errors" "fmt" - "github.com/spf13/cobra" - - "github.com/up9inc/mizu/cli/config" "github.com/up9inc/mizu/cli/mizu" + "regexp" + + "github.com/spf13/cobra" ) +type MizuTapOptions struct { + GuiPort uint16 + Namespace string + KubeConfigPath string + MizuImage string + MizuPodPort uint16 +} + + +var mizuTapOptions = &MizuTapOptions{} + var tapCmd = &cobra.Command{ - Use: "tap [PODNAME]", + Use: "tap [POD REGEX]", Short: "Record ingoing traffic of a kubernetes pod", Long: `Record the ingoing traffic of a kubernetes pod. Supported protocols are HTTP and gRPC.`, RunE: func(cmd *cobra.Command, args []string) error { if len(args) == 0 { - return errors.New("PODNAME argument is required") + return errors.New("POD REGEX argument is required") } else if len(args) > 1 { - return errors.New("Unexpected number of arguments") + return errors.New("unexpected number of arguments") } - podName := args[0] + regex, err := regexp.Compile(args[0]) + if err != nil { + return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], err)) + } - mizu.Run(podName) + RunMizuTap(regex, mizuTapOptions) return nil }, } @@ -31,9 +45,9 @@ var tapCmd = &cobra.Command{ func init() { rootCmd.AddCommand(tapCmd) - tapCmd.Flags().Uint16VarP(&config.Configuration.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver") - tapCmd.Flags().StringVarP(&config.Configuration.Namespace, "namespace", "n", "", "Namespace selector") - tapCmd.Flags().StringVarP(&config.Configuration.KubeConfigPath, "kubeconfig", "k", "", "Path to kubeconfig file") - tapCmd.Flags().StringVarP(&config.Configuration.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector") - tapCmd.Flags().Uint16VarP(&config.Configuration.MizuPodPort, "mizu-port", "", 8899, "Port which mizu cli will attempt to forward from the mizu collector pod") + tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver") + tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector") + tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file") + tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector") + tapCmd.Flags().Uint16VarP(&mizuTapOptions.MizuPodPort, "mizu-port", "", 8899, "Port which mizu cli will attempt to forward from the mizu collector pod") } diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go new file mode 100644 index 000000000..040957ecb --- /dev/null +++ b/cli/cmd/tapRunner.go @@ -0,0 +1,287 @@ +package cmd + +import ( + "context" + "fmt" + "os" + "os/signal" + "regexp" + "syscall" + "time" + + core "k8s.io/api/core/v1" + + "github.com/up9inc/mizu/cli/debounce" + "github.com/up9inc/mizu/cli/kubernetes" + "github.com/up9inc/mizu/cli/mizu" +) + +var mizuServiceAccountExists bool +var aggregatorService *core.Service + +const ( + updateTappersDelay = 5 * time.Second +) + +var currentlyTappedPods []core.Pod + +func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { + kubernetesProvider := kubernetes.NewProvider(tappingOptions.KubeConfigPath, tappingOptions.Namespace) + + defer cleanUpMizuResources(kubernetesProvider) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() // cancel will be called when this function exits + + if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegexQuery); err != nil { + return + } else { + currentlyTappedPods = matchingPods + } + + nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(ctx, kubernetesProvider, currentlyTappedPods) + if err != nil { + return + } + + if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil { + return + } + + go portForwardApiPod(ctx, kubernetesProvider, cancel, tappingOptions) // TODO convert this to job for built in pod ttl or have the running app handle this + go watchPodsForTapping(ctx, kubernetesProvider, cancel, podRegexQuery, tappingOptions) + go syncApiStatus(ctx, cancel, tappingOptions) + + //block until exit signal or error + waitForFinish(ctx, cancel) + + // TODO handle incoming traffic from tapper using a channel +} + +func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error { + if err := createMizuAggregator(ctx, kubernetesProvider, tappingOptions); err != nil { + return err + } + + if err := createMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil { + return err + } + + return nil +} + +func createMizuAggregator(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappingOptions *MizuTapOptions) error { + var err error + + mizuServiceAccountExists = createRBACIfNecessary(ctx, kubernetesProvider) + _, err = kubernetesProvider.CreateMizuAggregatorPod(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.MizuImage, mizuServiceAccountExists) + if err != nil { + fmt.Printf("Error creating mizu collector pod: %v\n", err) + return err + } + + aggregatorService, err = kubernetesProvider.CreateService(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, mizu.AggregatorPodName) + if err != nil { + fmt.Printf("Error creating mizu collector service: %v\n", err) + return err + } + + return nil +} + +func createMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error { + if err := kubernetesProvider.ApplyMizuTapperDaemonSet( + ctx, + mizu.ResourcesNamespace, + mizu.TapperDaemonSetName, + tappingOptions.MizuImage, + mizu.TapperPodName, + fmt.Sprintf("%s.%s.svc.cluster.local", aggregatorService.Name, aggregatorService.Namespace), + nodeToTappedPodIPMap, + mizuServiceAccountExists, + ); err != nil { + fmt.Printf("Error creating mizu tapper daemonset: %v\n", err) + return err + } + + return nil +} + +func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) { + fmt.Printf("\nRemoving mizu resources\n") + + removalCtx, _ := context.WithTimeout(context.Background(), 5 * time.Second) + if err := kubernetesProvider.RemovePod(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil { + fmt.Printf("Error removing Pod %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err); + } + if err := kubernetesProvider.RemoveService(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil { + fmt.Printf("Error removing Service %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err); + } + if err := kubernetesProvider.RemoveDaemonSet(removalCtx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil { + fmt.Printf("Error removing DaemonSet %s in namespace %s: %s (%v,%+v)\n", mizu.TapperDaemonSetName, mizu.ResourcesNamespace, err, err, err); + } +} + +func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, podRegex *regexp.Regexp, tappingOptions *MizuTapOptions) { + added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, kubernetesProvider.Namespace), podRegex) + + restartTappers := func() { + if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegex); err != nil { + fmt.Printf("Error getting pods by regex: %s (%v,%+v)\n", err, err, err) + cancel() + } else { + currentlyTappedPods = matchingPods + } + + nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(ctx, kubernetesProvider, currentlyTappedPods) + if err != nil { + fmt.Printf("Error building node to ips map: %s (%v,%+v)\n", err, err, err) + cancel() + } + + if err := createMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil { + fmt.Printf("Error updating daemonset: %s (%v,%+v)\n", err, err, err) + cancel() + } + } + restartTappersDebouncer := debounce.NewDebouncer(updateTappersDelay, restartTappers) + + for { + select { + case newTarget := <- added: + fmt.Printf("+%s\n", newTarget.Name) + + case removedTarget := <- removed: + fmt.Printf("-%s\n", removedTarget.Name) + restartTappersDebouncer.SetOn() + + case modifiedTarget := <- modified: + // Act only if the modified pod has already obtained an IP address. + // After filtering for IPs, on a normal pod restart this includes the following events: + // - Pod deletion + // - Pod reaches start state + // - Pod reaches ready state + // Ready/unready transitions might also trigger this event. + if modifiedTarget.Status.PodIP != "" { + restartTappersDebouncer.SetOn() + } + + case <- errorChan: + // TODO: Does this also perform cleanup? + cancel() + + case <- ctx.Done(): + return + } + } +} + +func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, tappingOptions *MizuTapOptions) { + podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.AggregatorPodName)) + added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, mizu.ResourcesNamespace), podExactRegex) + isPodReady := false + var portForward *kubernetes.PortForward + for { + select { + case <- added: + continue + case <- removed: + fmt.Printf("%s removed\n", mizu.AggregatorPodName) + cancel() + return + case modifiedPod := <- modified: + if modifiedPod.Status.Phase == "Running" && !isPodReady { + isPodReady = true + var err error + portForward, err = kubernetes.NewPortForward(kubernetesProvider, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.GuiPort, tappingOptions.MizuPodPort, cancel) + fmt.Printf("Web interface is now available at http://localhost:%d\n", tappingOptions.GuiPort) + if err != nil { + fmt.Printf("error forwarding port to pod %s\n", err) + cancel() + } + } + + case <- time.After(25 * time.Second): + if !isPodReady { + fmt.Printf("error: %s pod was not ready in time", mizu.AggregatorPodName) + cancel() + } + + case <- errorChan: + cancel() + + case <- ctx.Done(): + if portForward != nil { + portForward.Stop() + } + return + } + } +} + +func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool { + mizuRBACExists, err := kubernetesProvider.DoesMizuRBACExist(ctx, mizu.ResourcesNamespace) + if err != nil { + fmt.Printf("warning: could not ensure mizu rbac resources exist %v\n", err) + return false + } + if !mizuRBACExists { + var versionString = mizu.Version + if mizu.GitCommitHash != "" { + versionString += "-" + mizu.GitCommitHash + } + err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.ResourcesNamespace, versionString) + if err != nil { + fmt.Printf("warning: could not create mizu rbac resources %v\n", err) + return false + } + } + return true +} + +func getNodeHostToTappedPodIpsMap(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappedPods []core.Pod) (map[string][]string, error) { + nodeToTappedPodIPMap := make(map[string][]string, 0) + for _, pod := range tappedPods { + existingList := nodeToTappedPodIPMap[pod.Spec.NodeName] + if existingList == nil { + nodeToTappedPodIPMap[pod.Spec.NodeName] = []string {pod.Status.PodIP} + } else { + nodeToTappedPodIPMap[pod.Spec.NodeName] = append(nodeToTappedPodIPMap[pod.Spec.NodeName], pod.Status.PodIP) + } + } + return nodeToTappedPodIPMap, nil +} + +func waitForFinish(ctx context.Context, cancel context.CancelFunc) { + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + // block until ctx cancel is called or termination signal is received + select { + case <- ctx.Done(): + break + case <- sigChan: + cancel() + } +} + +func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOptions *MizuTapOptions) { + controlSocket, err := mizu.CreateControlSocket(fmt.Sprintf("ws://localhost:%d/ws", tappingOptions.GuiPort)) + if err != nil { + fmt.Printf("error establishing control socket connection %s\n", err) + cancel() + } + + for { + select { + case <- ctx.Done(): + return + default: + err = controlSocket.SendNewTappedPodsListMessage(currentlyTappedPods) + if err != nil { + fmt.Printf("error Sending message via control socket %s\n", err) + } + time.Sleep(10 * time.Second) + } + } + +} diff --git a/cli/cmd/version.go b/cli/cmd/version.go index 16a068bca..11a348a27 100644 --- a/cli/cmd/version.go +++ b/cli/cmd/version.go @@ -11,7 +11,7 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print version info", RunE: func(cmd *cobra.Command, args []string) error { - fmt.Printf("%s %s\n", mizu.Version, mizu.GitCommitHash) + fmt.Printf("%s (%s) %s\n", mizu.Version, mizu.Branch, mizu.GitCommitHash) return nil }, } diff --git a/cli/cmd/view.go b/cli/cmd/view.go index 19fdfea44..aa4aa928d 100644 --- a/cli/cmd/view.go +++ b/cli/cmd/view.go @@ -2,7 +2,6 @@ package cmd import ( "fmt" - "github.com/spf13/cobra" ) diff --git a/cli/config/config.go b/cli/config/config.go deleted file mode 100644 index 7fd0825f8..000000000 --- a/cli/config/config.go +++ /dev/null @@ -1,11 +0,0 @@ -package config - -type Options struct { - GuiPort uint16 - Namespace string - KubeConfigPath string - MizuImage string - MizuPodPort uint16 -} - -var Configuration = &Options{} diff --git a/cli/debounce/debounce.go b/cli/debounce/debounce.go new file mode 100644 index 000000000..7772591ef --- /dev/null +++ b/cli/debounce/debounce.go @@ -0,0 +1,42 @@ +package debounce + +import ( + "time" +) + +func NewDebouncer(timeout time.Duration, callback func()) *Debouncer { + var debouncer Debouncer + debouncer.setTimeout(timeout) + debouncer.setCallback(callback) + return &debouncer +} + +type Debouncer struct { + callback func() + running bool + timeout time.Duration + timer *time.Timer +} + +func (d *Debouncer) setTimeout(timeout time.Duration) { + // TODO: Return err if d.running + d.timeout = timeout +} + +func (d *Debouncer) setCallback(callback func()) { + callbackWrapped := func() { + callback() + d.running = false + } + + d.callback = callbackWrapped +} + +func (d *Debouncer) SetOn() { + if d.running == true { + return + } + + d.running = true + d.timer = time.AfterFunc(d.timeout, d.callback) +} diff --git a/cli/go.mod b/cli/go.mod index c19ade7b4..9d2a699f7 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -3,8 +3,12 @@ module github.com/up9inc/mizu/cli go 1.16 require ( + github.com/gorilla/websocket v1.4.2 github.com/spf13/cobra v1.1.3 + github.com/up9inc/mizu/shared v0.0.0 k8s.io/api v0.21.0 k8s.io/apimachinery v0.21.0 k8s.io/client-go v0.21.0 ) + +replace github.com/up9inc/mizu/shared v0.0.0 => ../shared diff --git a/cli/go.sum b/cli/go.sum index dd1710de5..9610fbffe 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -31,6 +31,7 @@ github.com/Azure/go-autorest/autorest/adal v0.9.5 h1:Y3bBUV4rTuxenJJs41HU3qmqsb+ github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= @@ -152,6 +153,7 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index b0c9f3ae3..348068b4c 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -3,12 +3,21 @@ package kubernetes import ( _ "bytes" "context" + "encoding/json" "errors" "fmt" + + "path/filepath" + "regexp" + + applyconfapp "k8s.io/client-go/applyconfigurations/apps/v1" + applyconfmeta "k8s.io/client-go/applyconfigurations/meta/v1" + applyconfcore "k8s.io/client-go/applyconfigurations/core/v1" core "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth/azure" @@ -19,8 +28,6 @@ import ( "k8s.io/client-go/tools/clientcmd" _ "k8s.io/client-go/tools/portforward" "k8s.io/client-go/util/homedir" - "path/filepath" - "strings" ) type Provider struct { @@ -31,7 +38,8 @@ type Provider struct { } const ( - serviceAccountName = "mizu-service-account" + serviceAccountName = "mizu-service-account" + fieldManagerName = "mizu-manager" ) func NewProvider(kubeConfigPath string, overrideNamespace string) *Provider { @@ -77,48 +85,31 @@ func (provider *Provider) GetPods(ctx context.Context, namespace string) { fmt.Printf("There are %d pods in Namespace %s\n", len(pods.Items), namespace) } -func (provider *Provider) CreateMizuPod(ctx context.Context, namespace string, podName string, podImage string, tappedPodNamespace string, tappedPodName string, linkServiceAccount bool) (*core.Pod, error) { - tappedPod, err := provider.clientSet.CoreV1().Pods(tappedPodNamespace).Get(ctx, tappedPodName, metav1.GetOptions{}) - if err != nil { - panic(err.Error()) - } - - podIps := make([]string, len(tappedPod.Status.PodIPs)) - for ii, podIp := range tappedPod.Status.PodIPs { - podIps[ii] = podIp.IP - } - podIpsString := strings.Join(podIps, ",") - - privileged := true +func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace string, podName string, podImage string, linkServiceAccount bool) (*core.Pod, error) { pod := &core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, Namespace: namespace, + Labels: map[string]string{"app": podName}, }, Spec: core.PodSpec{ - HostNetwork: true, // very important to make passive tapper see traffic Containers: []core.Container{ { Name: podName, Image: podImage, ImagePullPolicy: core.PullAlways, - SecurityContext: &core.SecurityContext{ - Privileged: &privileged, // must be privileged to get node level traffic - }, + Command: []string {"./mizuagent", "--aggregator"}, Env: []core.EnvVar{ { Name: "HOST_MODE", Value: "1", }, - { - Name: "TAPPED_ADDRESSES", - Value: podIpsString, - }, }, }, }, + DNSPolicy: "ClusterFirstWithHostNet", TerminationGracePeriodSeconds: new(int64), - NodeSelector: map[string]string{"kubernetes.io/hostname": tappedPod.Spec.NodeName}, + // Affinity: TODO: define node selector for all relevant nodes for this mizu instance }, } //define the service account only when it exists to prevent pod crash @@ -128,6 +119,21 @@ func (provider *Provider) CreateMizuPod(ctx context.Context, namespace string, p return provider.clientSet.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) } +func (provider *Provider) CreateService(ctx context.Context, namespace string, serviceName string, appLabelValue string) (*core.Service, error) { + service := core.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: namespace, + }, + Spec: core.ServiceSpec{ + Ports: []core.ServicePort {{TargetPort: intstr.FromInt(8899), Port: 80}}, + Type: core.ServiceTypeClusterIP, + Selector: map[string]string{"app": appLabelValue}, + }, + } + return provider.clientSet.CoreV1().Services(namespace).Create(ctx, &service, metav1.CreateOptions{}) +} + func (provider *Provider) DoesMizuRBACExist(ctx context.Context, namespace string) (bool, error){ serviceAccount, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccountName, metav1.GetOptions{}) @@ -200,8 +206,77 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string , return nil } -func (provider *Provider) RemovePod(ctx context.Context, namespace string, podName string) { - provider.clientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{}) +func (provider *Provider) RemovePod(ctx context.Context, namespace string, podName string) error { + return provider.clientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{}) +} + +func (provider *Provider) RemoveService(ctx context.Context, namespace string, serviceName string) error { + return provider.clientSet.CoreV1().Services(namespace).Delete(ctx, serviceName, metav1.DeleteOptions{}) +} + +func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, daemonSetName string) error { + return provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{}) +} + +func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, aggregatorPodIp string, nodeToTappedPodIPMap map[string][]string, linkServiceAccount bool) error { + nodeToTappedPodIPMapJsonStr, err := json.Marshal(nodeToTappedPodIPMap) + if err != nil { + return err + } + + privileged := true + agentContainer := applyconfcore.Container() + agentContainer.WithName(tapperPodName) + agentContainer.WithImage(podImage) + agentContainer.WithImagePullPolicy(core.PullAlways) + agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(privileged)) + agentContainer.WithCommand("./mizuagent", "-i", "any", "--tap", "--hardump", "--aggregator-address", fmt.Sprintf("ws://%s/wsTapper", aggregatorPodIp)) + agentContainer.WithEnv( + applyconfcore.EnvVar().WithName("HOST_MODE").WithValue("1"), + applyconfcore.EnvVar().WithName("AGGREGATOR_ADDRESS").WithValue(aggregatorPodIp), + applyconfcore.EnvVar().WithName("TAPPED_ADDRESSES_PER_HOST").WithValue(string(nodeToTappedPodIPMapJsonStr)), + ) + agentContainer.WithEnv( + applyconfcore.EnvVar().WithName("NODE_NAME").WithValueFrom( + applyconfcore.EnvVarSource().WithFieldRef( + applyconfcore.ObjectFieldSelector().WithAPIVersion("v1").WithFieldPath("spec.nodeName"), + ), + ), + ) + + podSpec := applyconfcore.PodSpec().WithHostNetwork(true).WithDNSPolicy("ClusterFirstWithHostNet").WithTerminationGracePeriodSeconds(0) + if linkServiceAccount { + podSpec.WithServiceAccountName(serviceAccountName) + } + podSpec.WithContainers(agentContainer) + + + podTemplate := applyconfcore.PodTemplateSpec() + podTemplate.WithLabels(map[string]string{"app": tapperPodName}) + podTemplate.WithSpec(podSpec) + + labelSelector := applyconfmeta.LabelSelector() + labelSelector.WithMatchLabels(map[string]string{"app": tapperPodName}) + + daemonSet := applyconfapp.DaemonSet(daemonSetName, namespace) + daemonSet.WithSpec(applyconfapp.DaemonSetSpec().WithSelector(labelSelector).WithTemplate(podTemplate)) + + _, err = provider.clientSet.AppsV1().DaemonSets(namespace).Apply(ctx, daemonSet, metav1.ApplyOptions{FieldManager: fieldManagerName}) + return err +} + +func (provider *Provider) GetAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp) ([]core.Pod, error) { + pods, err := provider.clientSet.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + matchingPods := make([]core.Pod, 0) + for _, pod := range pods.Items { + if regex.MatchString(pod.Name) { + matchingPods = append(matchingPods, pod) + } + } + return matchingPods, err } func getClientSet(config *restclient.Config) *kubernetes.Clientset { diff --git a/cli/mizu/consts.go b/cli/mizu/consts.go index 48b998a0b..22532d0e6 100644 --- a/cli/mizu/consts.go +++ b/cli/mizu/consts.go @@ -7,5 +7,8 @@ var ( ) const ( - MizuResourcesNamespace = "default" + ResourcesNamespace = "default" + TapperDaemonSetName = "mizu-tapper-daemon-set" + AggregatorPodName = "mizu-collector" + TapperPodName = "mizu-tapper" ) diff --git a/cli/mizu/controlSocket.go b/cli/mizu/controlSocket.go new file mode 100644 index 000000000..f35b9377c --- /dev/null +++ b/cli/mizu/controlSocket.go @@ -0,0 +1,42 @@ +package mizu + +import ( + "encoding/json" + "github.com/gorilla/websocket" + "github.com/up9inc/mizu/shared" + core "k8s.io/api/core/v1" + "time" +) + +type ControlSocket struct { + connection *websocket.Conn +} + +func CreateControlSocket(socketServerAddress string) (*ControlSocket, error) { + connection, err := shared.ConnectToSocketServer(socketServerAddress, 30, 2 * time.Second, true) + if err != nil { + return nil, err + } else { + return &ControlSocket{connection: connection}, nil + } +} + +func (controlSocket *ControlSocket) SendNewTappedPodsListMessage(pods []core.Pod) error { + podInfos := make([]shared.PodInfo, 0) + for _, pod := range pods { + podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace}) + } + tapStatus := shared.TapStatus{Pods: podInfos} + socketMessage := shared.CreateWebSocketStatusMessage(tapStatus) + + jsonMessage, err := json.Marshal(socketMessage) + if err != nil { + return err + } + err = controlSocket.connection.WriteMessage(websocket.TextMessage, jsonMessage) + if err != nil { + return err + } + + return nil +} diff --git a/debug.Dockerfile b/debug.Dockerfile index 2dab3c6f7..d0dc3102b 100644 --- a/debug.Dockerfile +++ b/debug.Dockerfile @@ -1,7 +1,7 @@ # creates image in which mizu api is remotely debuggable using delve FROM node:14-slim AS site-build -WORKDIR /ui-build +WORKDIR /app/ui-build COPY ui . RUN npm i @@ -15,14 +15,16 @@ ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64 RUN apk add libpcap-dev gcc g++ make # Move to api working directory (/api-build). -WORKDIR /api-build +WORKDIR /app/api-build COPY api/go.mod api/go.sum ./ +COPY shared/go.mod shared/go.mod ../shared/ RUN go mod download # cheap trick to make the build faster (As long as go.mod wasn't changes) RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get # Copy and build api code +COPY shared ../shared COPY api . RUN go build -gcflags="all=-N -l" -o mizuagent . @@ -33,10 +35,10 @@ RUN apk add bash libpcap-dev tcpdump WORKDIR /app # Copy binary and config files from /build to root folder of scratch container. -COPY --from=builder ["/api-build/mizuagent", "."] -COPY --from=site-build ["/ui-build/build", "site"] +COPY --from=builder ["/app/api-build/mizuagent", "."] +COPY --from=site-build ["/app/ui-build/build", "site"] # install remote debugging tool RUN go get github.com/go-delve/delve/cmd/dlv -CMD ["sh", "-c", "dlv --headless=true --listen=:2345 --log --api-version=2 --accept-multiclient exec ./mizuagent -- -i any -hardump -targets ${TAPPED_ADDRESSES}"] +CMD ["sh", "-c", "dlv --headless=true --listen=:2345 --log --api-version=2 --accept-multiclient exec ./mizuagent -- --aggregator"] diff --git a/shared/go.mod b/shared/go.mod new file mode 100644 index 000000000..d1afa338c --- /dev/null +++ b/shared/go.mod @@ -0,0 +1,5 @@ +module github.com/up9inc/mizu/shared + +go 1.16 + +require github.com/gorilla/websocket v1.4.2 diff --git a/shared/go.sum b/shared/go.sum new file mode 100644 index 000000000..85efffd99 --- /dev/null +++ b/shared/go.sum @@ -0,0 +1,2 @@ +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/shared/models.go b/shared/models.go new file mode 100644 index 000000000..dba9e1731 --- /dev/null +++ b/shared/models.go @@ -0,0 +1,35 @@ +package shared + +type WebSocketMessageType string +const ( + WebSocketMessageTypeEntry WebSocketMessageType = "entry" + WebSocketMessageTypeTappedEntry WebSocketMessageType = "tappedEntry" + WebSocketMessageTypeUpdateStatus WebSocketMessageType = "status" +) + +type WebSocketMessageMetadata struct { + MessageType WebSocketMessageType `json:"messageType,omitempty"` +} + +type WebSocketStatusMessage struct { + *WebSocketMessageMetadata + TappingStatus TapStatus `json:"tappingStatus"` +} + +type TapStatus struct { + Pods []PodInfo `json:"pods"` +} + +type PodInfo struct { + Namespace string `json:"namespace"` + Name string `json:"name"` +} + +func CreateWebSocketStatusMessage(tappingStatus TapStatus) WebSocketStatusMessage { + return WebSocketStatusMessage{ + WebSocketMessageMetadata: &WebSocketMessageMetadata{ + MessageType: WebSocketMessageTypeUpdateStatus, + }, + TappingStatus: tappingStatus, + } +} diff --git a/shared/socket_client.go b/shared/socket_client.go new file mode 100644 index 000000000..2faeabf5c --- /dev/null +++ b/shared/socket_client.go @@ -0,0 +1,39 @@ +package shared + +import ( + "fmt" + "github.com/gorilla/websocket" + "time" +) + +const ( + DEFAULT_SOCKET_RETRIES = 3 + DEFAULT_SOCKET_RETRY_SLEEP_TIME = time.Second * 10 +) + +func ConnectToSocketServer(address string, retries int, retrySleepTime time.Duration, hideTimeoutErrors bool) (*websocket.Conn, error) { + var err error + var connection *websocket.Conn + try := 0 + + // Connection to server fails if client pod is up before server. + // Retries solve this issue. + for try < retries { + connection, _, err = websocket.DefaultDialer.Dial(address, nil) + if err != nil { + try++ + if !hideTimeoutErrors { + fmt.Printf("Failed connecting to websocket server: %s, (%v,%+v)\n", err, err, err) + } + } else { + break + } + time.Sleep(retrySleepTime) + } + + if err != nil { + return nil, err + } + + return connection, nil +} diff --git a/ui/src/components/HarPage.tsx b/ui/src/components/HarPage.tsx index 6e937bcff..11edf80fa 100644 --- a/ui/src/components/HarPage.tsx +++ b/ui/src/components/HarPage.tsx @@ -8,6 +8,7 @@ import {HAREntryDetailed} from "./HarEntryDetailed"; import playIcon from './assets/play.svg'; import pauseIcon from './assets/pause.svg'; import variables from './style/variables.module.scss'; +import {StatusBar} from "./StatusBar"; const useLayoutStyles = makeStyles(() => ({ details: { @@ -49,6 +50,8 @@ export const HarPage: React.FC = () => { const [statusFilter, setStatusFilter] = useState([]); const [pathFilter, setPathFilter] = useState(""); + const [tappingStatus, setTappingStatus] = useState(null); + const ws = useRef(null); const openWebSocket = () => { @@ -60,23 +63,37 @@ export const HarPage: React.FC = () => { if(ws.current) { ws.current.onmessage = e => { if(!e?.data) return; - const entry = JSON.parse(e.data); - if(connection === ConnectionStatus.Paused) { - setNoMoreDataBottom(false) - return; + const message = JSON.parse(e.data); + + switch (message.messageType) { + case "entry": + const entry = message.data + if(connection === ConnectionStatus.Paused) { + setNoMoreDataBottom(false) + return; + } + if(!focusedEntryId) setFocusedEntryId(entry.id) + let newEntries = [...entries]; + if(entries.length === 1000) { + newEntries = newEntries.splice(1); + setNoMoreDataTop(false); + } + setEntries([...newEntries, entry]) + break + case "status": + setTappingStatus(message.tappingStatus); + break + default: + console.error(`unsupported websocket message type, Got: ${message.messageType}`) } - if(!focusedEntryId) setFocusedEntryId(entry.id) - let newEntries = [...entries]; - if(entries.length === 1000) { - newEntries = newEntries.splice(1); - setNoMoreDataTop(false); - } - setEntries([...newEntries, entry]) } } useEffect(() => { openWebSocket(); + fetch(`http://localhost:8899/api/tapStatus`) + .then(response => response.json()) + .then(data => setTappingStatus(data)); }, []); @@ -155,6 +172,7 @@ export const HarPage: React.FC = () => { {selectedHarEntry && } } + {tappingStatus?.pods != null && } ) }; diff --git a/ui/src/components/StatusBar.tsx b/ui/src/components/StatusBar.tsx new file mode 100644 index 000000000..ee09369c5 --- /dev/null +++ b/ui/src/components/StatusBar.tsx @@ -0,0 +1,28 @@ +import './style/StatusBar.sass'; +import React from "react"; + +export interface TappingStatusPod { + name: string; + namespace: string; +} + +export interface TappingStatus { + pods: TappingStatusPod[]; +} + +export interface Props { + tappingStatus: TappingStatus +} + +const pluralize = (noun: string, amount: number) => { + return `${noun}${amount != 1 ? 's' : ''}` +} + +export const StatusBar: React.FC = ({tappingStatus}) => { + const uniqueNamespaces = Array.from(new Set(tappingStatus.pods.map(pod => pod.namespace))); + const amountOfPods = tappingStatus.pods.length; + + return
+ {`Tapping ${amountOfPods} ${pluralize('pod', amountOfPods)} in ${pluralize('namespace', uniqueNamespaces.length)} ${uniqueNamespaces.join(", ")}`} +
; +} diff --git a/ui/src/components/style/HarPage.sass b/ui/src/components/style/HarPage.sass index 90f8b5aea..32025a45b 100644 --- a/ui/src/components/style/HarPage.sass +++ b/ui/src/components/style/HarPage.sass @@ -6,7 +6,13 @@ flex-direction: column overflow: hidden flex-grow: 1 - height: calc(100vh - 100px) + height: calc(100vh - 80px) + + .harPageHeader + padding: 20px 24px + display: flex + align-items: center + background-color: $header-background-color .harPageHeader padding: 20px 24px diff --git a/ui/src/components/style/StatusBar.sass b/ui/src/components/style/StatusBar.sass new file mode 100644 index 000000000..470166b25 --- /dev/null +++ b/ui/src/components/style/StatusBar.sass @@ -0,0 +1,20 @@ +@import 'variables.module.scss' + +.StatusBar + position: absolute + transform: translate(-50%, -3px) + left: 50% + z-index: 9999 + min-width: 200px + height: 32px + background: $blue-color + color: $light-blue-color + border-bottom-left-radius: 8px + border-bottom-right-radius: 8px + top: 0 + display: flex + align-items: center + padding: 2px 10px + user-select: none + font-size: 14px + opacity: 0.8