From 8636a4731e9e4c5090c84c33789e3473089cf150 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Wed, 6 Oct 2021 17:16:47 +0300 Subject: [PATCH] fixed ignored user agents (#322) --- acceptanceTests/tap_test.go | 110 ++++++++++++++++++ acceptanceTests/testsUtils.go | 15 +++ agent/main.go | 50 ++------ tap/extensions/http/handlers.go | 5 + tap/extensions/http/sensitive_data_cleaner.go | 24 ++++ 5 files changed, 165 insertions(+), 39 deletions(-) diff --git a/acceptanceTests/tap_test.go b/acceptanceTests/tap_test.go index 0f04ea6a8..e5dd33855 100644 --- a/acceptanceTests/tap_test.go +++ b/acceptanceTests/tap_test.go @@ -762,6 +762,116 @@ func TestTapRegexMasking(t *testing.T) { } } +func TestTapIgnoredUserAgents(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + tapCmdArgs := getDefaultTapCommandArgs() + + tapNamespace := getDefaultTapNamespace() + tapCmdArgs = append(tapCmdArgs, tapNamespace...) + + ignoredUserAgentValue := "ignore" + tapCmdArgs = append(tapCmdArgs, "--set", fmt.Sprintf("tap.ignored-user-agents=%v", ignoredUserAgentValue)) + + tapCmd := exec.Command(cliPath, tapCmdArgs...) + t.Logf("running command: %v", tapCmd.String()) + + t.Cleanup(func() { + if err := cleanupCommand(tapCmd); err != nil { + t.Logf("failed to cleanup tap command, err: %v", err) + } + }) + + if err := tapCmd.Start(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + apiServerUrl := getApiServerUrl(defaultApiServerPort) + + if err := waitTapPodsReady(apiServerUrl); err != nil { + t.Errorf("failed to start tap pods on time, err: %v", err) + return + } + + proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName) + + ignoredUserAgentCustomHeader := "Ignored-User-Agent" + headers := map[string]string {"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""} + for i := 0; i < defaultEntriesCount; i++ { + if _, requestErr := executeHttpGetRequestWithHeaders(fmt.Sprintf("%v/get", proxyUrl), headers); requestErr != nil { + t.Errorf("failed to send proxy request, err: %v", requestErr) + return + } + } + + for i := 0; i < defaultEntriesCount; i++ { + if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil { + t.Errorf("failed to send proxy request, err: %v", requestErr) + return + } + } + + ignoredUserAgentsCheckFunc := func() error { + timestamp := time.Now().UnixNano() / int64(time.Millisecond) + + entriesUrl := fmt.Sprintf("%v/api/entries?limit=%v&operator=lt×tamp=%v", apiServerUrl, defaultEntriesCount * 2, timestamp) + requestResult, requestErr := executeHttpGetRequest(entriesUrl) + if requestErr != nil { + return fmt.Errorf("failed to get entries, err: %v", requestErr) + } + + entries := requestResult.([]interface{}) + if len(entries) == 0 { + return fmt.Errorf("unexpected entries result - Expected more than 0 entries") + } + + for _, entryInterface := range entries { + entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, entryInterface.(map[string]interface{})["id"]) + requestResult, requestErr = executeHttpGetRequest(entryUrl) + if requestErr != nil { + return fmt.Errorf("failed to get entry, err: %v", requestErr) + } + + data := requestResult.(map[string]interface{})["data"].(map[string]interface{}) + entryJson := data["entry"].(string) + + var entry map[string]interface{} + if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil { + return fmt.Errorf("failed to parse entry, err: %v", parseErr) + } + + entryRequest := entry["request"].(map[string]interface{}) + entryPayload := entryRequest["payload"].(map[string]interface{}) + entryDetails := entryPayload["details"].(map[string]interface{}) + + entryHeaders := entryDetails["headers"].([]interface{}) + for _, headerInterface := range entryHeaders { + header := headerInterface.(map[string]interface{}) + if header["name"].(string) != ignoredUserAgentCustomHeader { + continue + } + + return fmt.Errorf("unexpected result - user agent is not ignored") + } + } + + return nil + } + if err := retriesExecute(shortRetriesCount, ignoredUserAgentsCheckFunc); err != nil { + t.Errorf("%v", err) + return + } +} + func TestTapDumpLogs(t *testing.T) { if testing.Short() { t.Skip("ignored acceptance test") diff --git a/acceptanceTests/testsUtils.go b/acceptanceTests/testsUtils.go index 19359b066..45a2c0f8b 100644 --- a/acceptanceTests/testsUtils.go +++ b/acceptanceTests/testsUtils.go @@ -172,6 +172,21 @@ func executeHttpRequest(response *http.Response, requestErr error) (interface{}, return jsonBytesToInterface(data) } +func executeHttpGetRequestWithHeaders(url string, headers map[string]string) (interface{}, error) { + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + for headerKey, headerValue := range headers { + request.Header.Add(headerKey, headerValue) + } + + client := &http.Client{} + response, requestErr := client.Do(request) + return executeHttpRequest(response, requestErr) +} + func executeHttpGetRequest(url string) (interface{}, error) { response, requestErr := http.Get(url) return executeHttpRequest(response, requestErr) diff --git a/agent/main.go b/agent/main.go index 71efb4709..53ae74fa7 100644 --- a/agent/main.go +++ b/agent/main.go @@ -4,6 +4,13 @@ import ( "encoding/json" "flag" "fmt" + "github.com/gin-contrib/static" + "github.com/gin-gonic/gin" + "github.com/gorilla/websocket" + "github.com/romana/rlog" + "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/tap" + tapApi "github.com/up9inc/mizu/tap/api" "io/ioutil" "log" "mizuserver/pkg/api" @@ -18,15 +25,6 @@ import ( "path/filepath" "plugin" "sort" - "strings" - - "github.com/gin-contrib/static" - "github.com/gin-gonic/gin" - "github.com/gorilla/websocket" - "github.com/romana/rlog" - "github.com/up9inc/mizu/shared" - "github.com/up9inc/mizu/tap" - tapApi "github.com/up9inc/mizu/tap/api" ) var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API") @@ -59,7 +57,7 @@ func main() { filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem) tap.StartPassiveTapper(tapOpts, outputItemsChannel, extensions, filteringOptions) - go filterItems(outputItemsChannel, filteredOutputItemsChannel, filteringOptions) + go filterItems(outputItemsChannel, filteredOutputItemsChannel) go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap) hostApi(nil) @@ -90,7 +88,7 @@ func main() { outputItemsChannel := make(chan *tapApi.OutputChannelItem) filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem) - go filterItems(outputItemsChannel, filteredOutputItemsChannel, filteringOptions) + go filterItems(outputItemsChannel, filteredOutputItemsChannel) go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap) hostApi(outputItemsChannel) @@ -98,7 +96,7 @@ func main() { outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000) filteredHarChannel := make(chan *tapApi.OutputChannelItem) - go filterItems(outputItemsChannel, filteredHarChannel, filteringOptions) + go filterItems(outputItemsChannel, filteredHarChannel) go api.StartReadingEntries(filteredHarChannel, harsDir, extensionsMap) hostApi(nil) } @@ -242,42 +240,16 @@ func getTrafficFilteringOptions() *tapApi.TrafficFilteringOptions { return &filteringOptions } -func filterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem, filterOptions *tapApi.TrafficFilteringOptions) { +func filterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem) { for message := range inChannel { if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) { continue } - // TODO: move this to tappers https://up9.atlassian.net/browse/TRA-3441 - if isIgnoredUserAgent(message, filterOptions.IgnoredUserAgents) { - continue - } outChannel <- message } } -func isIgnoredUserAgent(item *tapApi.OutputChannelItem, userAgentsToIgnore []string) bool { - if item.Protocol.Name != "http" { - return false - } - - request := item.Pair.Request.Payload.(map[string]interface{}) - reqDetails := request["details"].(map[string]interface{}) - - for _, header := range reqDetails["headers"].([]interface{}) { - h := header.(map[string]interface{}) - if strings.ToLower(h["name"].(string)) == "user-agent" { - for _, userAgent := range userAgentsToIgnore { - if strings.Contains(strings.ToLower(h["value"].(string)), strings.ToLower(userAgent)) { - return true - } - } - return false - } - } - return false -} - func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tapApi.OutputChannelItem) { if connection == nil { panic("Websocket connection is nil") diff --git a/tap/extensions/http/handlers.go b/tap/extensions/http/handlers.go index 7c71d7691..d41bf3e54 100644 --- a/tap/extensions/http/handlers.go +++ b/tap/extensions/http/handlers.go @@ -14,9 +14,14 @@ import ( ) func filterAndEmit(item *api.OutputChannelItem, emitter api.Emitter, options *api.TrafficFilteringOptions) { + if IsIgnoredUserAgent(item, options) { + return + } + if !options.DisableRedaction { FilterSensitiveData(item, options) } + emitter.Emit(item) } diff --git a/tap/extensions/http/sensitive_data_cleaner.go b/tap/extensions/http/sensitive_data_cleaner.go index 75edbe78e..9ea0914bf 100644 --- a/tap/extensions/http/sensitive_data_cleaner.go +++ b/tap/extensions/http/sensitive_data_cleaner.go @@ -25,6 +25,30 @@ var personallyIdentifiableDataFields = []string{"token", "authorization", "authe "zip", "zipcode", "address", "country", "firstname", "lastname", "middlename", "fname", "lname", "birthdate"} +func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) bool { + if item.Protocol.Name != "http" { + return false + } + + request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request) + + for headerKey, headerValues := range request.Header { + if strings.ToLower(headerKey) == "user-agent" { + for _, userAgent := range options.IgnoredUserAgents { + for _, headerValue := range headerValues { + if strings.Contains(strings.ToLower(headerValue), strings.ToLower(userAgent)) { + return true + } + } + } + + return false + } + } + + return false +} + func FilterSensitiveData(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) { request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request) response := item.Pair.Response.Payload.(HTTPPayload).Data.(*http.Response)