From 77783a58e9c8fedea98faa7bf5ff890a56003567 Mon Sep 17 00:00:00 2001 From: Roee Gadot Date: Wed, 18 Aug 2021 08:28:22 +0300 Subject: [PATCH] roee changes - trying to fix agent to work with the "api" object) - ***still not working*** --- Dockerfile | 7 +- agent/go.mod | 2 + agent/go.sum | 82 +++++++++++++++- agent/main.go | 107 ++++++++++----------- agent/pkg/api/main.go | 120 +++++++++++++----------- agent/pkg/api/socket_server_handlers.go | 6 +- agent/pkg/models/models.go | 84 ++++++++++++++++- agent/start.sh | 2 - cli/kubernetes/provider.go | 1 - shared/go.mod | 5 +- shared/go.sum | 8 +- shared/models.go | 2 +- tap/api/api.go | 9 +- tap/extensions/amqp/main.go | 7 +- tap/extensions/http/main.go | 86 +++++++++-------- tap/extensions/http/matcher.go | 19 ++-- tap/extensions/kafka/main.go | 7 +- tap/passive_tapper.go | 30 ++---- tap/tcp_stream_factory.go | 1 + 19 files changed, 383 insertions(+), 202 deletions(-) delete mode 100755 agent/start.sh diff --git a/Dockerfile b/Dockerfile index c19b1805f..0d76fcf52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ WORKDIR /app/agent-build COPY agent/go.mod agent/go.sum ./ COPY shared/go.mod shared/go.mod ../shared/ COPY tap/go.mod tap/go.mod ../tap/ +COPY tap/api ../tap/api RUN go mod download # cheap trick to make the build faster (As long as go.mod wasn't changes) RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get @@ -38,6 +39,9 @@ RUN go build -ldflags="-s -w \ -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \ -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent . +## TODO: need to be changed to copy the build script ./build_extensions.sh and run it +WORKDIR /app/agent-build/extensions +COPY tap/extensions/*.so . FROM alpine:3.13.5 @@ -46,10 +50,9 @@ WORKDIR /app # Copy binary and config files from /build to root folder of scratch container. COPY --from=builder ["/app/agent-build/mizuagent", "."] +COPY --from=builder ["/app/agent-build/extensions", "extensions"] COPY --from=site-build ["/app/ui-build/build", "site"] -COPY agent/start.sh . - # gin-gonic runs in debug mode without this ENV GIN_MODE=release diff --git a/agent/go.mod b/agent/go.mod index 2d1012eb3..15884fac8 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -17,7 +17,9 @@ require ( github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 github.com/up9inc/mizu/shared v0.0.0 github.com/up9inc/mizu/tap v0.0.0 + github.com/up9inc/mizu/tap/api v0.0.0 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 + go.mongodb.org/mongo-driver v1.7.1 gorm.io/driver/sqlite v1.1.4 gorm.io/gorm v1.21.8 k8s.io/api v0.21.0 diff --git a/agent/go.sum b/agent/go.sum index 12760e2dd..7095d96cd 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -42,6 +42,7 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -96,6 +97,31 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E= github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -120,6 +146,7 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -156,19 +183,26 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -180,6 +214,8 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ= @@ -192,6 +228,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= @@ -205,41 +242,63 @@ github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1p github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI= github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.mongodb.org/mongo-driver v1.7.1 h1:jwqTeEM3x6L9xDXrCxN0Hbg7vdGfPBOTIkr0+/LYZDA= +go.mongodb.org/mongo-driver v1.7.1/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= @@ -296,8 +355,9 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758 h1:aEpZnXcAmXkd6AvLb2OPt+EN1Zu/8Ne3pCqPjja5PXY= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -308,16 +368,22 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -337,8 +403,9 @@ golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe h1:WdX7u8s3yOigWAhHEaDl8r9G+4XwFQEQFtBMYyN+kXQ= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= @@ -348,8 +415,10 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -361,9 +430,13 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -460,8 +533,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM= gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw= gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= diff --git a/agent/main.go b/agent/main.go index cce894095..7c133a811 100644 --- a/agent/main.go +++ b/agent/main.go @@ -4,6 +4,13 @@ import ( "encoding/json" "flag" "fmt" + "github.com/gin-contrib/static" + "github.com/gin-gonic/gin" + "github.com/gorilla/websocket" + "github.com/romana/rlog" + "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/tap" + tapApi "github.com/up9inc/mizu/tap/api" "mizuserver/pkg/api" "mizuserver/pkg/models" "mizuserver/pkg/routes" @@ -11,13 +18,6 @@ import ( "net/http" "os" "os/signal" - - "github.com/gin-contrib/static" - "github.com/gin-gonic/gin" - "github.com/gorilla/websocket" - "github.com/romana/rlog" - "github.com/up9inc/mizu/shared" - "github.com/up9inc/mizu/tap" ) var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API") @@ -38,12 +38,12 @@ func main() { if *standaloneMode { api.StartResolving(*namespace) - harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts) - filteredHarChannel := make(chan *tap.OutputChannelItem) + filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem) + tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel) - go filterHarItems(harOutputChannel, filteredHarChannel, getTrafficFilteringOptions()) - go api.StartReadingEntries(filteredHarChannel, nil) - go api.StartReadingOutbound(outboundLinkOutputChannel) + // go filterHarItems(harOutputChannel, filteredOutputItemsChannel, getTrafficFilteringOptions()) + go api.StartReadingEntries(filteredOutputItemsChannel, nil) + // go api.StartReadingOutbound(outboundLinkOutputChannel) hostApi(nil) } else if *tapperMode { @@ -57,22 +57,23 @@ func main() { rlog.Infof("Filtering for the following authorities: %v", tap.GetFilterIPs()) } - harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts) - + // harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts) + filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem) + tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel) socketConnection, err := shared.ConnectToSocketServer(*apiServerAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false) if err != nil { panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err)) } - go pipeTapChannelToSocket(socketConnection, harOutputChannel) - go pipeOutboundLinksChannelToSocket(socketConnection, outboundLinkOutputChannel) + go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel) + // go pipeOutboundLinksChannelToSocket(socketConnection, outboundLinkOutputChannel) } else if *apiServerMode { api.StartResolving(*namespace) - socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000) - filteredHarChannel := make(chan *tap.OutputChannelItem) + socketHarOutChannel := make(chan *tapApi.OutputChannelItem, 1000) + filteredHarChannel := make(chan *tapApi.OutputChannelItem) - go filterHarItems(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions()) + // go filterHarItems(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions()) go api.StartReadingEntries(filteredHarChannel, nil) hostApi(socketHarOutChannel) @@ -85,7 +86,7 @@ func main() { rlog.Info("Exiting") } -func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) { +func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) { app := gin.Default() app.GET("/echo", func(c *gin.Context) { @@ -93,7 +94,7 @@ func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) { }) eventHandlers := api.RoutesEventHandlers{ - SocketHarOutChannel: socketHarOutputChannel, + SocketOutChannel: socketHarOutputChannel, } app.Use(static.ServeRoot("/", "./site")) @@ -148,41 +149,41 @@ func getTrafficFilteringOptions() *shared.TrafficFilteringOptions { return &filteringOptions } -var userAgentsToFilter = []string{"kube-probe", "prometheus"} +// var userAgentsToFilter = []string{"kube-probe", "prometheus"} -func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) { - for message := range inChannel { - // if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) { - // continue - // } - // TODO: move this to tappers https://up9.atlassian.net/browse/TRA-3441 - if filterOptions.HideHealthChecks && isHealthCheckByUserAgent(message) { - continue - } +//func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) { +// for message := range inChannel { +// if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) { +// continue +// } +// // TODO: move this to tappers https://up9.atlassian.net/browse/TRA-3441 +// if filterOptions.HideHealthChecks && isHealthCheckByUserAgent(message) { +// continue +// } +// +// if !filterOptions.DisableRedaction { +// sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions) +// } +// +// outChannel <- message +// } +//} - // if !filterOptions.DisableRedaction { - // sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions) - // } +//func isHealthCheckByUserAgent(message *tap.OutputChannelItem) bool { +// // for _, header := range message.HarEntry.Request.Headers { +// // if strings.ToLower(header.Name) == "user-agent" { +// // for _, userAgent := range userAgentsToFilter { +// // if strings.Contains(strings.ToLower(header.Value), userAgent) { +// // return true +// // } +// // } +// // return false +// // } +// // } +// return false +//} - outChannel <- message - } -} - -func isHealthCheckByUserAgent(message *tap.OutputChannelItem) bool { - // for _, header := range message.HarEntry.Request.Headers { - // if strings.ToLower(header.Name) == "user-agent" { - // for _, userAgent := range userAgentsToFilter { - // if strings.Contains(strings.ToLower(header.Value), userAgent) { - // return true - // } - // } - // return false - // } - // } - return false -} - -func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) { +func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tapApi.OutputChannelItem) { if connection == nil { panic("Websocket connection is nil") } diff --git a/agent/pkg/api/main.go b/agent/pkg/api/main.go index 3e55b529a..f4365dac3 100644 --- a/agent/pkg/api/main.go +++ b/agent/pkg/api/main.go @@ -5,7 +5,10 @@ import ( "context" "encoding/json" "fmt" + "go.mongodb.org/mongo-driver/bson/primitive" + "mizuserver/pkg/database" "mizuserver/pkg/holder" + "net/http" "net/url" "os" "path" @@ -16,6 +19,7 @@ import ( "github.com/google/martian/har" "github.com/romana/rlog" "github.com/up9inc/mizu/tap" + tapApi "github.com/up9inc/mizu/tap/api" "mizuserver/pkg/models" "mizuserver/pkg/resolver" @@ -46,7 +50,7 @@ func StartResolving(namespace string) { holder.SetResolver(res) } -func StartReadingEntries(harChannel <-chan *tap.OutputChannelItem, workingDir *string) { +func StartReadingEntries(harChannel <-chan *tapApi.OutputChannelItem, workingDir *string) { if workingDir != nil && *workingDir != "" { startReadingFiles(*workingDir) } else { @@ -100,14 +104,18 @@ func startReadingFiles(workingDir string) { } } -func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) { +func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem) { if outputItems == nil { panic("Channel of captured messages is nil") } - // for item := range outputItems { - // saveHarToDb(item.HarEntry, item.ConnectionInfo) - // } + for item := range outputItems { + if harEntry, err := models.NewEntry(item.Data.Request.Orig.(*http.Request), item.Data.Request.CaptureTime, item.Data.Response.Orig.(*http.Response), item.Data.Response.CaptureTime); err == nil { + saveHarToDb(harEntry, item.ConnectionInfo) + } else { + rlog.Errorf("Error when creating HTTP entry") + } + } } func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) { @@ -117,59 +125,59 @@ func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) { } } -// func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) { -// entryBytes, _ := json.Marshal(entry) -// serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL) -// entryId := primitive.NewObjectID().Hex() -// var ( -// resolvedSource string -// resolvedDestination string -// ) -// if k8sResolver != nil { -// unresolvedSource := connectionInfo.ClientIP -// resolvedSource = k8sResolver.Resolve(unresolvedSource) -// if resolvedSource == "" { -// rlog.Debugf("Cannot find resolved name to source: %s\n", unresolvedSource) -// if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" { -// return -// } -// } -// unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort) -// resolvedDestination = k8sResolver.Resolve(unresolvedDestination) -// if resolvedDestination == "" { -// rlog.Debugf("Cannot find resolved name to dest: %s\n", unresolvedDestination) -// if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" { -// return -// } -// } -// } +func saveHarToDb(entry *har.Entry, connectionInfo *tapApi.ConnectionInfo) { + entryBytes, _ := json.Marshal(entry) + serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL) + entryId := primitive.NewObjectID().Hex() + var ( + resolvedSource string + resolvedDestination string + ) + if k8sResolver != nil { + unresolvedSource := connectionInfo.ClientIP + resolvedSource = k8sResolver.Resolve(unresolvedSource) + if resolvedSource == "" { + rlog.Debugf("Cannot find resolved name to source: %s\n", unresolvedSource) + if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" { + return + } + } + unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort) + resolvedDestination = k8sResolver.Resolve(unresolvedDestination) + if resolvedDestination == "" { + rlog.Debugf("Cannot find resolved name to dest: %s\n", unresolvedDestination) + if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" { + return + } + } + } -// mizuEntry := models.MizuEntry{ -// EntryId: entryId, -// Entry: string(entryBytes), // simple way to store it and not convert to bytes -// Service: serviceName, -// Url: entry.Request.URL, -// Path: urlPath, -// Method: entry.Request.Method, -// Status: entry.Response.Status, -// RequestSenderIp: connectionInfo.ClientIP, -// Timestamp: entry.StartedDateTime.UnixNano() / int64(time.Millisecond), -// ResolvedSource: resolvedSource, -// ResolvedDestination: resolvedDestination, -// IsOutgoing: connectionInfo.IsOutgoing, -// } -// mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry) -// database.CreateEntry(&mizuEntry) + mizuEntry := models.MizuEntry{ + EntryId: entryId, + Entry: string(entryBytes), // simple way to store it and not convert to bytes + Service: serviceName, + Url: entry.Request.URL, + Path: urlPath, + Method: entry.Request.Method, + Status: entry.Response.Status, + RequestSenderIp: connectionInfo.ClientIP, + Timestamp: entry.StartedDateTime.UnixNano() / int64(time.Millisecond), + ResolvedSource: resolvedSource, + ResolvedDestination: resolvedDestination, + IsOutgoing: connectionInfo.IsOutgoing, + } + mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry) + database.CreateEntry(&mizuEntry) -// baseEntry := models.BaseEntryDetails{} -// if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil { -// return -// } -// baseEntry.Rules = models.RunValidationRulesState(*entry, serviceName) -// baseEntry.Latency = entry.Timings.Receive -// baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry) -// BroadcastToBrowserClients(baseEntryBytes) -// } + baseEntry := models.BaseEntryDetails{} + if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil { + return + } + baseEntry.Rules = models.RunValidationRulesState(*entry, serviceName) + baseEntry.Latency = entry.Timings.Receive + baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry) + BroadcastToBrowserClients(baseEntryBytes) +} func getServiceNameFromUrl(inputUrl string) (string, string) { parsed, err := url.Parse(inputUrl) diff --git a/agent/pkg/api/socket_server_handlers.go b/agent/pkg/api/socket_server_handlers.go index f6b4627c8..7afe1c1c6 100644 --- a/agent/pkg/api/socket_server_handlers.go +++ b/agent/pkg/api/socket_server_handlers.go @@ -3,6 +3,7 @@ package api import ( "encoding/json" "fmt" + tapApi "github.com/up9inc/mizu/tap/api" "mizuserver/pkg/models" "mizuserver/pkg/providers" "mizuserver/pkg/up9" @@ -10,7 +11,6 @@ import ( "github.com/romana/rlog" "github.com/up9inc/mizu/shared" - "github.com/up9inc/mizu/tap" ) var browserClientSocketUUIDs = make([]int, 0) @@ -18,7 +18,7 @@ var socketListLock = sync.Mutex{} type RoutesEventHandlers struct { EventHandlers - SocketHarOutChannel chan<- *tap.OutputChannelItem + SocketOutChannel chan<- *tapApi.OutputChannelItem } func init() { @@ -71,7 +71,7 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) { if err != nil { rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err) } else { - h.SocketHarOutChannel <- tappedEntryMessage.Data + h.SocketOutChannel <- tappedEntryMessage.Data } case shared.WebSocketMessageTypeUpdateStatus: var statusMessage shared.WebSocketStatusMessage diff --git a/agent/pkg/models/models.go b/agent/pkg/models/models.go index 13013f819..922027ead 100644 --- a/agent/pkg/models/models.go +++ b/agent/pkg/models/models.go @@ -1,7 +1,15 @@ package models import ( + "bytes" "encoding/json" + "errors" + "fmt" + tapApi "github.com/up9inc/mizu/tap/api" + "io/ioutil" + "net/http" + "strconv" + "strings" "mizuserver/pkg/rules" "mizuserver/pkg/utils" @@ -141,7 +149,7 @@ type WebSocketEntryMessage struct { type WebSocketTappedEntryMessage struct { *shared.WebSocketMessageMetadata - Data *tap.OutputChannelItem + Data *tapApi.OutputChannelItem } type WebsocketOutboundLinkMessage struct { @@ -159,7 +167,7 @@ func CreateBaseEntryWebSocketMessage(base *BaseEntryDetails) ([]byte, error) { return json.Marshal(message) } -func CreateWebsocketTappedEntryMessage(base *tap.OutputChannelItem) ([]byte, error) { +func CreateWebsocketTappedEntryMessage(base *tapApi.OutputChannelItem) ([]byte, error) { message := &WebSocketTappedEntryMessage{ WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{ MessageType: shared.WebSocketMessageTypeTappedEntry, @@ -222,3 +230,75 @@ func RunValidationRulesState(harEntry har.Entry, service string) ApplicableRules ar := NewApplicableRules(statusPolicyToSend, latency) return ar } + +func NewEntry(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time) (*har.Entry, error) { + harRequest, err := har.NewRequest(request, false) + if err != nil { + fmt.Printf("Failed converting request to HAR %s (%v,%+v)", err, err, err) + return nil, errors.New("failed converting request to HAR") + } + + // For requests with multipart/form-data or application/x-www-form-urlencoded Content-Type, + // martian/har will parse the request body and place the parameters in harRequest.PostData.Params + // instead of harRequest.PostData.Text (as the HAR spec requires it). + // Mizu currently only looks at PostData.Text. Therefore, instead of letting martian/har set the content of + // PostData, always copy the request body to PostData.Text. + if request.ContentLength > 0 { + reqBody, err := ioutil.ReadAll(request.Body) + if err != nil { + fmt.Printf("Failed converting request to HAR %s (%v,%+v)", err, err, err) + return nil, errors.New("failed reading request body") + } + request.Body = ioutil.NopCloser(bytes.NewReader(reqBody)) + harRequest.PostData.Text = string(reqBody) + } + + harResponse, err := har.NewResponse(response, true) + if err != nil { + fmt.Printf("Failed converting response to HAR %s (%v,%+v)", err, err, err) + return nil, errors.New("failed converting response to HAR") + } + + if harRequest.PostData != nil && strings.HasPrefix(harRequest.PostData.MimeType, "application/grpc") { + // Force HTTP/2 gRPC into HAR template + + harRequest.URL = fmt.Sprintf("%s://%s%s", request.Header.Get(":scheme"), request.Header.Get(":authority"), request.Header.Get(":path")) + + status, err := strconv.Atoi(response.Header.Get(":status")) + if err != nil { + fmt.Printf("Failed converting status to int %s (%v,%+v)", err, err, err) + return nil, errors.New("failed converting response status to int for HAR") + } + harResponse.Status = status + } else { + // Martian copies http.Request.URL.String() to har.Request.URL, which usually contains the path. + // However, according to the HAR spec, the URL field needs to be the absolute URL. + var scheme string + if request.URL.Scheme != "" { + scheme = request.URL.Scheme + } else { + scheme = "http" + } + harRequest.URL = fmt.Sprintf("%s://%s%s", scheme, request.Host, request.URL) + } + + totalTime := responseTime.Sub(requestTime).Round(time.Millisecond).Milliseconds() + if totalTime < 1 { + totalTime = 1 + } + + harEntry := har.Entry{ + StartedDateTime: time.Now().UTC(), + Time: totalTime, + Request: harRequest, + Response: harResponse, + Cache: &har.Cache{}, + Timings: &har.Timings{ + Send: -1, + Wait: -1, + Receive: totalTime, + }, + } + + return &harEntry, nil +} diff --git a/agent/start.sh b/agent/start.sh deleted file mode 100755 index 4b04b6d47..000000000 --- a/agent/start.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -./mizuagent -i any -hardump -targets ${TAPPED_ADDRESSES} diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index fcbb78502..221895ad4 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -577,7 +577,6 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac "./mizuagent", "-i", "any", "--tap", - "--hardump", "--api-server-address", fmt.Sprintf("ws://%s/wsTapper", apiServerPodIp), } if tapOutgoing { diff --git a/shared/go.mod b/shared/go.mod index 157d3e5fa..5e50ad375 100644 --- a/shared/go.mod +++ b/shared/go.mod @@ -3,8 +3,7 @@ module github.com/up9inc/mizu/shared go 1.16 require ( - github.com/google/martian v2.1.0+incompatible // indirect - github.com/gorilla/websocket v1.4.2 - github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect github.com/docker/go-units v0.4.0 + github.com/gorilla/websocket v1.4.2 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/shared/go.sum b/shared/go.sum index 498bce1d3..65c692cb5 100644 --- a/shared/go.sum +++ b/shared/go.sum @@ -1,8 +1,8 @@ -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/shared/models.go b/shared/models.go index 2a70fa212..18a51eea4 100644 --- a/shared/models.go +++ b/shared/models.go @@ -5,7 +5,7 @@ import ( "io/ioutil" "strings" - yaml "gopkg.in/yaml.v3" + "gopkg.in/yaml.v3" ) type WebSocketMessageType string diff --git a/tap/api/api.go b/tap/api/api.go index edc801714..d97a2f27e 100644 --- a/tap/api/api.go +++ b/tap/api/api.go @@ -41,8 +41,15 @@ type RequestResponsePair struct { Response GenericMessage `json:"response"` } +type OutputChannelItem struct { + Type string + Timestamp int64 + ConnectionInfo *ConnectionInfo + Data *RequestResponsePair +} + type Dissector interface { Register(*Extension) Ping() - Dissect(b *bufio.Reader, isClient bool, tcpID *TcpID, callback func(reqResPair *RequestResponsePair)) + Dissect(b *bufio.Reader, isClient bool, tcpID *TcpID) *OutputChannelItem } diff --git a/tap/extensions/amqp/main.go b/tap/extensions/amqp/main.go index e3f7b907b..3e92e078b 100644 --- a/tap/extensions/amqp/main.go +++ b/tap/extensions/amqp/main.go @@ -13,18 +13,19 @@ func init() { type dissecting string -func (d dissecting) Register(extension *api.Extension) { +func (g dissecting) Register(extension *api.Extension) { extension.Name = "amqp" extension.OutboundPorts = []string{"5671", "5672"} extension.InboundPorts = []string{} } -func (d dissecting) Ping() { +func (g dissecting) Ping() { log.Printf("pong AMQP\n") } -func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, callback func(reqResPair *api.RequestResponsePair)) { +func (g dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID) *api.OutputChannelItem { // TODO: Implement + return nil } var Dissector dissecting diff --git a/tap/extensions/http/main.go b/tap/extensions/http/main.go index 0c2c6d58f..b7cf3b44f 100644 --- a/tap/extensions/http/main.go +++ b/tap/extensions/http/main.go @@ -4,7 +4,10 @@ import ( "bufio" "fmt" "io" + "io/ioutil" "log" + "net/http" + "time" "github.com/up9inc/mizu/tap/api" ) @@ -20,60 +23,69 @@ func init() { type dissecting string -func (d dissecting) Register(extension *api.Extension) { - extension.Name = "http" +const ExtensionName = "http" + +func (g dissecting) Register(extension *api.Extension) { + extension.Name = ExtensionName extension.OutboundPorts = []string{"80", "8080", "443"} extension.InboundPorts = []string{} } -func (d dissecting) Ping() { +func (g dissecting) Ping() { log.Printf("pong HTTP\n") } -func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, Emit func(reqResPair *api.RequestResponsePair)) { - ident := fmt.Sprintf("%s->%s:%s->%s", tcpID.SrcIP, tcpID.DstIP, tcpID.SrcPort, tcpID.DstPort) - isHTTP2, err := checkIsHTTP2Connection(b, isClient) - if err != nil { - SilentError("HTTP/2-Prepare-Connection", "stream %s Failed to check if client is HTTP/2: %s (%v,%+v)", ident, err, err, err) - // Do something? - } - - var grpcAssembler *GrpcAssembler - if isHTTP2 { - err := prepareHTTP2Connection(b, isClient) - if err != nil { - SilentError("HTTP/2-Prepare-Connection-After-Check", "stream %s error: %s (%v,%+v)", ident, err, err, err) - } - grpcAssembler = createGrpcAssembler(b) - } - +func (g dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID) *api.OutputChannelItem { for { - if isHTTP2 { - err = handleHTTP2Stream(grpcAssembler, tcpID, Emit) + if isClient { + requestCounter++ + req, err := http.ReadRequest(b) if err == io.EOF || err == io.ErrUnexpectedEOF { - break + return nil } else if err != nil { - SilentError("HTTP/2", "stream %s error: %s (%v,%+v)", ident, err, err, err) - continue - } - } else if isClient { - err = handleHTTP1ClientStream(b, tcpID, Emit) - if err == io.EOF || err == io.ErrUnexpectedEOF { - break - } else if err != nil { - SilentError("HTTP-request", "stream %s Request error: %s (%v,%+v)", ident, err, err, err) - continue + log.Println("Error reading stream:", err) + } else { + body, _ := ioutil.ReadAll(req.Body) + req.Body.Close() + log.Printf("Received request: %+v with body: %+v\n", req, body) } + + ident := fmt.Sprintf( + "%s->%s %s->%s %d", + tcpID.SrcIP, + tcpID.DstIP, + tcpID.SrcPort, + tcpID.DstPort, + requestCounter, + ) + reqResMatcher.registerRequest(ident, req, time.Now()) } else { - err = handleHTTP1ServerStream(b, tcpID, Emit) + responseCounter++ + res, err := http.ReadResponse(b, nil) if err == io.EOF || err == io.ErrUnexpectedEOF { - break + return nil } else if err != nil { - SilentError("HTTP-response", "stream %s Response error: %s (%v,%+v)", ident, err, err, err) - continue + log.Println("Error reading stream:", err) + } else { + body, _ := ioutil.ReadAll(res.Body) + res.Body.Close() + log.Printf("Received response: %+v with body: %+v\n", res, body) + } + ident := fmt.Sprintf( + "%s->%s %s->%s %d", + tcpID.DstIP, + tcpID.SrcIP, + tcpID.DstPort, + tcpID.SrcPort, + responseCounter, + ) + reqResPair := reqResMatcher.registerResponse(ident, res, time.Now()) + if reqResPair != nil { + return reqResPair } } } + return nil } var Dissector dissecting diff --git a/tap/extensions/http/matcher.go b/tap/extensions/http/matcher.go index fa98038af..fb94cf11e 100644 --- a/tap/extensions/http/matcher.go +++ b/tap/extensions/http/matcher.go @@ -12,7 +12,7 @@ import ( var reqResMatcher = createResponseRequestMatcher() // global -// Key is {client_addr}:{client_port}->{dest_addr}:{dest_port} +// Key is {client_addr}:{client_port}->{dest_addr}:{dest_port}_{incremental_counter} type requestResponseMatcher struct { openMessagesMap sync.Map } @@ -22,7 +22,7 @@ func createResponseRequestMatcher() requestResponseMatcher { return *newMatcher } -func (matcher *requestResponseMatcher) registerRequest(ident string, request *http.Request, captureTime time.Time) *api.RequestResponsePair { +func (matcher *requestResponseMatcher) registerRequest(ident string, request *http.Request, captureTime time.Time) *api.OutputChannelItem { split := splitIdent(ident) key := genKey(split) // fmt.Printf(">>> request key: %v\n", key) @@ -49,7 +49,7 @@ func (matcher *requestResponseMatcher) registerRequest(ident string, request *ht return nil } -func (matcher *requestResponseMatcher) registerResponse(ident string, response *http.Response, captureTime time.Time) *api.RequestResponsePair { +func (matcher *requestResponseMatcher) registerResponse(ident string, response *http.Response, captureTime time.Time) *api.OutputChannelItem { split := splitIdent(ident) key := genKey(split) // fmt.Printf(">>> response key: %v\n", key) @@ -76,10 +76,15 @@ func (matcher *requestResponseMatcher) registerResponse(ident string, response * return nil } -func (matcher *requestResponseMatcher) preparePair(requestHTTPMessage *api.GenericMessage, responseHTTPMessage *api.GenericMessage) *api.RequestResponsePair { - return &api.RequestResponsePair{ - Request: *requestHTTPMessage, - Response: *responseHTTPMessage, +func (matcher *requestResponseMatcher) preparePair(requestHTTPMessage *api.GenericMessage, responseHTTPMessage *api.GenericMessage) *api.OutputChannelItem { + return &api.OutputChannelItem{ + Type: ExtensionName, + Timestamp: time.Now().UnixNano() / int64(time.Millisecond), + ConnectionInfo: nil, + Data: &api.RequestResponsePair{ + Request: *requestHTTPMessage, + Response: *responseHTTPMessage, + }, } } diff --git a/tap/extensions/kafka/main.go b/tap/extensions/kafka/main.go index 6aca561a3..4edb543b3 100644 --- a/tap/extensions/kafka/main.go +++ b/tap/extensions/kafka/main.go @@ -13,18 +13,19 @@ func init() { type dissecting string -func (d dissecting) Register(extension *api.Extension) { +func (g dissecting) Register(extension *api.Extension) { extension.Name = "kafka" extension.OutboundPorts = []string{"9092"} extension.InboundPorts = []string{} } -func (d dissecting) Ping() { +func (g dissecting) Ping() { log.Printf("pong Kafka\n") } -func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, callback func(reqResPair *api.RequestResponsePair)) { +func (g dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID) *api.OutputChannelItem { // TODO: Implement + return nil } var Dissector dissecting diff --git a/tap/passive_tapper.go b/tap/passive_tapper.go index f6a934a7b..3a917e379 100644 --- a/tap/passive_tapper.go +++ b/tap/passive_tapper.go @@ -90,8 +90,7 @@ var memprofile = flag.String("memprofile", "", "Write memory profile") var dumpToHar = flag.Bool("hardump", false, "Dump traffic to har files") var HarOutputDir = flag.String("hardir", "", "Directory in which to store output har files") var harEntriesPerFile = flag.Int("harentriesperfile", 200, "Number of max number of har entries to store in each file") - -var filter = flag.String("f", "tcp and (src port 80 or dst port 80)", "BPF filter for pcap") +var filter = flag.String("f", "tcp", "BPF filter for pcap") var statsTracker = StatsTracker{} @@ -128,9 +127,6 @@ var extensions []*api.Extension // global var allOutboundPorts []string // global var allInboundPorts []string // global -type OutputChannelItem struct { -} - /* minOutputLevel: Error will be printed only if outputLevel is above this value * t: key for errorsMap (counting errors) * s, a: arguments log.Printf @@ -193,22 +189,14 @@ func (c *Context) GetCaptureInfo() gopacket.CaptureInfo { return c.CaptureInfo } -func StartPassiveTapper(opts *TapOpts) (<-chan *OutputChannelItem, <-chan *OutboundLink) { +func StartPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem) () { hostMode = opts.HostMode - // var harWriter *HarWriter - // if *dumpToHar { - // harWriter = NewHarWriter(*HarOutputDir, *harEntriesPerFile) - // } - outboundLinkWriter := NewOutboundLinkWriter() + if GetMemoryProfilingEnabled() { + startMemoryProfiler() + } - go startPassiveTapper(outboundLinkWriter) - - // if harWriter != nil { - // return harWriter.OutChan, outboundLinkWriter.OutChan - // } - - return nil, nil + go startPassiveTapper(outputItems) } func startMemoryProfiler() { @@ -289,7 +277,7 @@ func loadExtensions() { log.Printf("allInboundPorts: %v\n", allInboundPorts) } -func startPassiveTapper(outboundLinkWriter *OutboundLinkWriter) { +func startPassiveTapper(outputItems chan *api.OutputChannelItem) { loadExtensions() log.SetFlags(log.LstdFlags | log.LUTC | log.Lshortfile) @@ -315,7 +303,9 @@ func startPassiveTapper(outboundLinkWriter *OutboundLinkWriter) { } // Set up assembly - streamFactory := &tcpStreamFactory{} + streamFactory := &tcpStreamFactory{ + OutputChannelItem: outputItems, + } streamPool := tcpassembly.NewStreamPool(streamFactory) assembler := tcpassembly.NewAssembler(streamPool) diff --git a/tap/tcp_stream_factory.go b/tap/tcp_stream_factory.go index 88c84f9b3..79ba4bb95 100644 --- a/tap/tcp_stream_factory.go +++ b/tap/tcp_stream_factory.go @@ -15,6 +15,7 @@ import ( type tcpStreamFactory struct { outbountLinkWriter *OutboundLinkWriter + OutputChannelItem chan *api.OutputChannelItem } const checkTLSPacketAmount = 100