From db427d91cc101cab20affaa3df2781f75fce8793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Mert=20Y=C4=B1ld=C4=B1ran?= Date: Wed, 9 Feb 2022 13:34:52 +0300 Subject: [PATCH] Add unit tests for HTTP dissector (#767) * Add unit tests for HTTP dissector * Ignore some fields on test environment * Remove Git patches * Don't have indent in the expected JSONs * Fix more issues and update expected JSONs * Refactor the test environment lookup * Add a Makefile * Include HTTP tests into the CI * Fix the linting errors * Fix another linting error * Update the expected JSONs * Sort `PostData.Params` params as well * Move expected JSONs into `expect/dissect` * Add `TestAnalyze` method * Add `TestRepresent` method * Add `TestRegister`, `TestMacros` and `TestPing` methods * Test extensions first * Remove expected JSONs * Remove `bin` directory and update `Makefile` rules * Update `.gitignore` * Fix skipping download functionality in the Makefile * Run `go mod tidy` * Fix the race condition in the tests * Revert "Test extensions first" This reverts commit b8350cf139e625610f2eb7b89e600396dbee0cbc. * Make `TEST_UPDATE` env lookup one-liner * Update .github/workflows/test.yml Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com> * Add a newline * Replace `ls` with `find` Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com> --- .github/workflows/test.yml | 16 +- .gitignore | 4 + Makefile | 3 + tap/api/api.go | 67 ++++++- tap/extensions/http/Makefile | 16 ++ tap/extensions/http/go.mod | 4 + tap/extensions/http/go.sum | 13 ++ tap/extensions/http/helpers.go | 5 + tap/extensions/http/main_test.go | 300 +++++++++++++++++++++++++++++++ 9 files changed, 424 insertions(+), 4 deletions(-) create mode 100644 tap/extensions/http/Makefile create mode 100644 tap/extensions/http/main_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8c7eaa59c..3b7fa5b81 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,19 +27,31 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v2 - - shell: bash + - name: Install libpcap + shell: bash run: | sudo apt-get install libpcap-dev + - id: 'auth' + uses: 'google-github-actions/auth@v0' + with: + credentials_json: '${{ secrets.GCR_JSON_KEY }}' + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v0' + - name: CLI Test run: make test-cli - name: Agent Test run: make test-agent - + - name: Shared Test run: make test-shared + - name: Extensions Test + run: make test-extensions + - name: Upload coverage to Codecov uses: codecov/codecov-action@v2 diff --git a/.gitignore b/.gitignore index 35366c80f..81261416e 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,7 @@ cypress.env.json */cypress/screenshots */cypress/videos */cypress/support + +# Ignore test data in extensions +tap/extensions/*/bin +tap/extensions/*/expect diff --git a/Makefile b/Makefile index 893593f9b..835d68636 100644 --- a/Makefile +++ b/Makefile @@ -101,5 +101,8 @@ test-agent: test-shared: @echo "running shared tests"; cd shared && $(MAKE) test +test-extensions: + @echo "running http tests"; cd tap/extensions/http && $(MAKE) test + acceptance-test: @echo "running acceptance tests"; cd acceptanceTests && $(MAKE) test diff --git a/tap/api/api.go b/tap/api/api.go index f468a021a..52c3cac11 100644 --- a/tap/api/api.go +++ b/tap/api/api.go @@ -8,12 +8,16 @@ import ( "fmt" "io/ioutil" "net/http" + "os" + "sort" "sync" "time" "github.com/google/martian/har" ) +const mizuTestEnvVar = "MIZU_TEST" + type Protocol struct { Name string `json:"name"` LongName string `json:"longName"` @@ -262,27 +266,86 @@ type HTTPWrapper struct { } func (h HTTPPayload) MarshalJSON() ([]byte, error) { + _, testEnvEnabled := os.LookupEnv(mizuTestEnvVar) switch h.Type { case TypeHttpRequest: harRequest, err := har.NewRequest(h.Data.(*http.Request), true) if err != nil { return nil, errors.New("Failed converting request to HAR") } + sort.Slice(harRequest.Headers, func(i, j int) bool { + if harRequest.Headers[i].Name < harRequest.Headers[j].Name { + return true + } + if harRequest.Headers[i].Name > harRequest.Headers[j].Name { + return false + } + return harRequest.Headers[i].Value < harRequest.Headers[j].Value + }) + sort.Slice(harRequest.QueryString, func(i, j int) bool { + if harRequest.QueryString[i].Name < harRequest.QueryString[j].Name { + return true + } + if harRequest.QueryString[i].Name > harRequest.QueryString[j].Name { + return false + } + return harRequest.QueryString[i].Value < harRequest.QueryString[j].Value + }) + if harRequest.PostData != nil { + sort.Slice(harRequest.PostData.Params, func(i, j int) bool { + if harRequest.PostData.Params[i].Name < harRequest.PostData.Params[j].Name { + return true + } + if harRequest.PostData.Params[i].Name > harRequest.PostData.Params[j].Name { + return false + } + return harRequest.PostData.Params[i].Value < harRequest.PostData.Params[j].Value + }) + } + if testEnvEnabled { + harRequest.URL = "" + } + var reqWrapper *HTTPRequestWrapper + if !testEnvEnabled { + reqWrapper = &HTTPRequestWrapper{Request: h.Data.(*http.Request)} + } return json.Marshal(&HTTPWrapper{ Method: harRequest.Method, Details: harRequest, - RawRequest: &HTTPRequestWrapper{Request: h.Data.(*http.Request)}, + RawRequest: reqWrapper, }) case TypeHttpResponse: harResponse, err := har.NewResponse(h.Data.(*http.Response), true) if err != nil { return nil, errors.New("Failed converting response to HAR") } + sort.Slice(harResponse.Headers, func(i, j int) bool { + if harResponse.Headers[i].Name < harResponse.Headers[j].Name { + return true + } + if harResponse.Headers[i].Name > harResponse.Headers[j].Name { + return false + } + return harResponse.Headers[i].Value < harResponse.Headers[j].Value + }) + sort.Slice(harResponse.Cookies, func(i, j int) bool { + if harResponse.Cookies[i].Name < harResponse.Cookies[j].Name { + return true + } + if harResponse.Cookies[i].Name > harResponse.Cookies[j].Name { + return false + } + return harResponse.Cookies[i].Value < harResponse.Cookies[j].Value + }) + var resWrapper *HTTPResponseWrapper + if !testEnvEnabled { + resWrapper = &HTTPResponseWrapper{Response: h.Data.(*http.Response)} + } return json.Marshal(&HTTPWrapper{ Method: "", Url: "", Details: harResponse, - RawResponse: &HTTPResponseWrapper{Response: h.Data.(*http.Response)}, + RawResponse: resWrapper, }) default: panic(fmt.Sprintf("HTTP payload cannot be marshaled: %v", h.Type)) diff --git a/tap/extensions/http/Makefile b/tap/extensions/http/Makefile new file mode 100644 index 000000000..2ad21ae65 --- /dev/null +++ b/tap/extensions/http/Makefile @@ -0,0 +1,16 @@ +skipbin := $$(find bin -mindepth 1 -maxdepth 1) +skipexpect := $$(find expect -mindepth 1 -maxdepth 1) + +test: test-pull-bin test-pull-expect + @MIZU_TEST=1 go test -v ./... -coverpkg=./... -race -coverprofile=coverage.out -covermode=atomic + +test-update: test-pull-bin + @MIZU_TEST=1 TEST_UPDATE=1 go test -v ./... -coverpkg=./... -coverprofile=coverage.out -covermode=atomic + +test-pull-bin: + @mkdir -p bin + @[ "${skipbin}" ] && echo "Skipping downloading BINs" || gsutil -m cp gs://static.up9.io/mizu/test-pcap/bin/http/\*.bin bin + +test-pull-expect: + @mkdir -p expect + @[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil -m cp -r gs://static.up9.io/mizu/test-pcap/expect/http/\* expect diff --git a/tap/extensions/http/go.mod b/tap/extensions/http/go.mod index 8da80be42..fc93b8fac 100644 --- a/tap/extensions/http/go.mod +++ b/tap/extensions/http/go.mod @@ -4,13 +4,17 @@ go 1.17 require ( github.com/beevik/etree v1.1.0 + github.com/stretchr/testify v1.7.0 github.com/up9inc/mizu/tap/api v0.0.0 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd ) require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/google/martian v2.1.0+incompatible // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/text v0.3.7 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api diff --git a/tap/extensions/http/go.sum b/tap/extensions/http/go.sum index 9da088156..78ff0cef0 100644 --- a/tap/extensions/http/go.sum +++ b/tap/extensions/http/go.sum @@ -1,7 +1,15 @@ github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -10,3 +18,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tap/extensions/http/helpers.go b/tap/extensions/http/helpers.go index 0c84178b4..8507af158 100644 --- a/tap/extensions/http/helpers.go +++ b/tap/extensions/http/helpers.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "reflect" + "sort" "strconv" "github.com/up9inc/mizu/tap/api" @@ -39,6 +40,10 @@ func mapSliceMergeRepeatedKeys(mapSlice []interface{}) (newMapSlice []interface{ newMapSlice = append(newMapSlice, h) } + sort.Slice(newMapSlice, func(i, j int) bool { + return newMapSlice[i].(map[string]interface{})["name"].(string) < newMapSlice[j].(map[string]interface{})["name"].(string) + }) + return } diff --git a/tap/extensions/http/main_test.go b/tap/extensions/http/main_test.go new file mode 100644 index 000000000..97cbc6430 --- /dev/null +++ b/tap/extensions/http/main_test.go @@ -0,0 +1,300 @@ +package http + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/up9inc/mizu/tap/api" +) + +const ( + binDir = "bin" + patternBin = "*_req.bin" + patternDissect = "*.json" + msgDissecting = "Dissecting:" + msgAnalyzing = "Analyzing:" + msgRepresenting = "Representing:" + respSuffix = "_res.bin" + expectDir = "expect" + dissectDir = "dissect" + analyzeDir = "analyze" + representDir = "represent" + testUpdate = "TEST_UPDATE" +) + +func TestRegister(t *testing.T) { + dissector := NewDissector() + extension := &api.Extension{} + dissector.Register(extension) + assert.Equal(t, "http", extension.Protocol.Name) + assert.NotNil(t, extension.MatcherMap) +} + +func TestMacros(t *testing.T) { + expectedMacros := map[string]string{ + "http": `proto.name == "http" and proto.version.startsWith("1")`, + "http2": `proto.name == "http" and proto.version == "2.0"`, + "grpc": `proto.name == "http" and proto.version == "2.0" and proto.macro == "grpc"`, + } + dissector := NewDissector() + macros := dissector.Macros() + assert.Equal(t, expectedMacros, macros) +} + +func TestPing(t *testing.T) { + dissector := NewDissector() + dissector.Ping() +} + +func TestDissect(t *testing.T) { + _, testUpdateEnabled := os.LookupEnv(testUpdate) + + expectDirDissect := path.Join(expectDir, dissectDir) + + if testUpdateEnabled { + os.RemoveAll(expectDirDissect) + err := os.MkdirAll(expectDirDissect, 0775) + assert.Nil(t, err) + } + + dissector := NewDissector() + paths, err := filepath.Glob(path.Join(binDir, patternBin)) + if err != nil { + log.Fatal(err) + } + + options := &api.TrafficFilteringOptions{ + IgnoredUserAgents: []string{}, + } + + for _, _path := range paths { + basePath := _path[:len(_path)-8] + + // Channel to verify the output + itemChannel := make(chan *api.OutputChannelItem) + var emitter api.Emitter = &api.Emitting{ + AppStats: &api.AppStats{}, + OutputChannel: itemChannel, + } + + var items []*api.OutputChannelItem + stop := make(chan bool) + + go func() { + for { + select { + case <-stop: + return + case item := <-itemChannel: + items = append(items, item) + } + } + }() + + // Stream level + counterPair := &api.CounterPair{ + Request: 0, + Response: 0, + } + superIdentifier := &api.SuperIdentifier{} + + // Request + pathClient := _path + fmt.Printf("%s %s\n", msgDissecting, pathClient) + fileClient, err := os.Open(pathClient) + assert.Nil(t, err) + + bufferClient := bufio.NewReader(fileClient) + tcpIDClient := &api.TcpID{ + SrcIP: "1", + DstIP: "2", + SrcPort: "1", + DstPort: "2", + } + err = dissector.Dissect(bufferClient, true, tcpIDClient, counterPair, &api.SuperTimer{}, superIdentifier, emitter, options) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + panic(err) + } + + // Response + pathServer := basePath + respSuffix + fmt.Printf("%s %s\n", msgDissecting, pathServer) + fileServer, err := os.Open(pathServer) + assert.Nil(t, err) + + bufferServer := bufio.NewReader(fileServer) + tcpIDServer := &api.TcpID{ + SrcIP: "2", + DstIP: "1", + SrcPort: "2", + DstPort: "1", + } + err = dissector.Dissect(bufferServer, false, tcpIDServer, counterPair, &api.SuperTimer{}, superIdentifier, emitter, options) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + panic(err) + } + + fileClient.Close() + fileServer.Close() + + pathExpect := path.Join(expectDirDissect, fmt.Sprintf("%s.json", basePath[4:])) + + time.Sleep(10 * time.Millisecond) + + stop <- true + + sort.Slice(items, func(i, j int) bool { + iMarshaled, err := json.Marshal(items[i]) + assert.Nil(t, err) + jMarshaled, err := json.Marshal(items[j]) + assert.Nil(t, err) + return len(iMarshaled) < len(jMarshaled) + }) + + marshaled, err := json.Marshal(items) + assert.Nil(t, err) + + if testUpdateEnabled { + if len(items) > 0 { + err = os.WriteFile(pathExpect, marshaled, 0644) + assert.Nil(t, err) + } + } else { + if _, err := os.Stat(pathExpect); errors.Is(err, os.ErrNotExist) { + assert.Len(t, items, 0) + } else { + expectedBytes, err := ioutil.ReadFile(pathExpect) + assert.Nil(t, err) + + assert.JSONEq(t, string(expectedBytes), string(marshaled)) + } + } + } +} + +func TestAnalyze(t *testing.T) { + _, testUpdateEnabled := os.LookupEnv(testUpdate) + + expectDirDissect := path.Join(expectDir, dissectDir) + expectDirAnalyze := path.Join(expectDir, analyzeDir) + + if testUpdateEnabled { + os.RemoveAll(expectDirAnalyze) + err := os.MkdirAll(expectDirAnalyze, 0775) + assert.Nil(t, err) + } + + dissector := NewDissector() + paths, err := filepath.Glob(path.Join(expectDirDissect, patternDissect)) + if err != nil { + log.Fatal(err) + } + + for _, _path := range paths { + fmt.Printf("%s %s\n", msgAnalyzing, _path) + + bytes, err := ioutil.ReadFile(_path) + assert.Nil(t, err) + + var items []*api.OutputChannelItem + err = json.Unmarshal(bytes, &items) + assert.Nil(t, err) + + var entries []*api.Entry + for _, item := range items { + entry := dissector.Analyze(item, "", "") + entries = append(entries, entry) + } + + pathExpect := path.Join(expectDirAnalyze, filepath.Base(_path)) + + marshaled, err := json.Marshal(entries) + assert.Nil(t, err) + + if testUpdateEnabled { + if len(entries) > 0 { + err = os.WriteFile(pathExpect, marshaled, 0644) + assert.Nil(t, err) + } + } else { + if _, err := os.Stat(pathExpect); errors.Is(err, os.ErrNotExist) { + assert.Len(t, items, 0) + } else { + expectedBytes, err := ioutil.ReadFile(pathExpect) + assert.Nil(t, err) + + assert.JSONEq(t, string(expectedBytes), string(marshaled)) + } + } + } +} + +func TestRepresent(t *testing.T) { + _, testUpdateEnabled := os.LookupEnv(testUpdate) + + expectDirAnalyze := path.Join(expectDir, analyzeDir) + expectDirRepresent := path.Join(expectDir, representDir) + + if testUpdateEnabled { + os.RemoveAll(expectDirRepresent) + err := os.MkdirAll(expectDirRepresent, 0775) + assert.Nil(t, err) + } + + dissector := NewDissector() + paths, err := filepath.Glob(path.Join(expectDirAnalyze, patternDissect)) + if err != nil { + log.Fatal(err) + } + + for _, _path := range paths { + fmt.Printf("%s %s\n", msgRepresenting, _path) + + bytes, err := ioutil.ReadFile(_path) + assert.Nil(t, err) + + var entries []*api.Entry + err = json.Unmarshal(bytes, &entries) + assert.Nil(t, err) + + var objects []string + for _, entry := range entries { + object, _, err := dissector.Represent(entry.Request, entry.Response) + assert.Nil(t, err) + objects = append(objects, string(object)) + } + + pathExpect := path.Join(expectDirRepresent, filepath.Base(_path)) + + marshaled, err := json.Marshal(objects) + assert.Nil(t, err) + + if testUpdateEnabled { + if len(objects) > 0 { + err = os.WriteFile(pathExpect, marshaled, 0644) + assert.Nil(t, err) + } + } else { + if _, err := os.Stat(pathExpect); errors.Is(err, os.ErrNotExist) { + assert.Len(t, objects, 0) + } else { + expectedBytes, err := ioutil.ReadFile(pathExpect) + assert.Nil(t, err) + + assert.JSONEq(t, string(expectedBytes), string(marshaled)) + } + } + } +}