From 7c159fffc0a4f0d29cbb71f6e1772b517293c69c Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Tue, 12 Jul 2022 10:19:24 +0300 Subject: [PATCH] Added redact using insertion filter (#1196) --- acceptanceTests/cypress.config.js | 1 - .../cypress/e2e/tests/RegexMasking.js | 7 - acceptanceTests/tap_test.go | 58 +---- cli/cmd/tap.go | 1 - cli/cmd/tapRunner.go | 17 +- cli/config/configStructs/tapConfig.go | 103 ++++++--- performance_analysis/run_tapper_benchmark.sh | 2 +- tap/api/options.go | 4 +- tap/extensions/http/handlers.go | 4 - tap/extensions/http/sensitive_data_cleaner.go | 205 ------------------ 10 files changed, 78 insertions(+), 324 deletions(-) delete mode 100644 acceptanceTests/cypress/e2e/tests/RegexMasking.js diff --git a/acceptanceTests/cypress.config.js b/acceptanceTests/cypress.config.js index aaa620128..506441092 100644 --- a/acceptanceTests/cypress.config.js +++ b/acceptanceTests/cypress.config.js @@ -11,7 +11,6 @@ module.exports = defineConfig({ testUrl: 'http://localhost:8899/', redactHeaderContent: 'User-Header[REDACTED]', redactBodyContent: '{ "User": "[REDACTED]" }', - regexMaskingBodyContent: '[REDACTED]', greenFilterColor: 'rgb(210, 250, 210)', redFilterColor: 'rgb(250, 214, 220)', bodyJsonClass: '.hljs', diff --git a/acceptanceTests/cypress/e2e/tests/RegexMasking.js b/acceptanceTests/cypress/e2e/tests/RegexMasking.js deleted file mode 100644 index 9f3f20fac..000000000 --- a/acceptanceTests/cypress/e2e/tests/RegexMasking.js +++ /dev/null @@ -1,7 +0,0 @@ -import {isValueExistsInElement} from "../testHelpers/TrafficHelper"; - -it('Loading Mizu', function () { - cy.visit(Cypress.env('testUrl')); -}); - -isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), Cypress.env('bodyJsonClass')); diff --git a/acceptanceTests/tap_test.go b/acceptanceTests/tap_test.go index b71210bbb..a48069b4c 100644 --- a/acceptanceTests/tap_test.go +++ b/acceptanceTests/tap_test.go @@ -2,10 +2,8 @@ package acceptanceTests import ( "archive/zip" - "bytes" "fmt" "io/ioutil" - "net/http" "os/exec" "path" "strings" @@ -343,7 +341,7 @@ func TestTapRedact(t *testing.T) { tapNamespace := GetDefaultTapNamespace() tapCmdArgs = append(tapCmdArgs, tapNamespace...) - tapCmdArgs = append(tapCmdArgs, "--redact") + tapCmdArgs = append(tapCmdArgs, "--redact", "--set", "tap.redact-patterns.request-headers=User-Header", "--set", "tap.redact-patterns.request-body=User") tapCmd := exec.Command(cliPath, tapCmdArgs...) t.Logf("running command: %v", tapCmd.String()) @@ -429,60 +427,6 @@ func TestTapNoRedact(t *testing.T) { RunCypressTests(t, "npx cypress run --spec \"cypress/e2e/tests/NoRedact.js\"") } -func TestTapRegexMasking(t *testing.T) { - if testing.Short() { - t.Skip("ignored acceptance test") - } - - cliPath, cliPathErr := GetCliPath() - if cliPathErr != nil { - t.Errorf("failed to get cli path, err: %v", cliPathErr) - return - } - - tapCmdArgs := GetDefaultTapCommandArgs() - - tapNamespace := GetDefaultTapNamespace() - tapCmdArgs = append(tapCmdArgs, tapNamespace...) - - tapCmdArgs = append(tapCmdArgs, "--redact") - - tapCmdArgs = append(tapCmdArgs, "-r", "Mizu") - - tapCmd := exec.Command(cliPath, tapCmdArgs...) - t.Logf("running command: %v", tapCmd.String()) - - t.Cleanup(func() { - if err := CleanupCommand(tapCmd); err != nil { - t.Logf("failed to cleanup tap command, err: %v", err) - } - }) - - if err := tapCmd.Start(); err != nil { - t.Errorf("failed to start tap command, err: %v", err) - return - } - - apiServerUrl := GetApiServerUrl(DefaultApiServerPort) - - if err := WaitTapPodsReady(apiServerUrl); err != nil { - t.Errorf("failed to start tap pods on time, err: %v", err) - return - } - - proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName) - for i := 0; i < DefaultEntriesCount; i++ { - response, requestErr := http.Post(fmt.Sprintf("%v/post", proxyUrl), "text/plain", bytes.NewBufferString("Mizu")) - if _, requestErr = ExecuteHttpRequest(response, requestErr); requestErr != nil { - t.Errorf("failed to send proxy request, err: %v", requestErr) - return - } - } - - RunCypressTests(t, "npx cypress run --spec \"cypress/e2e/tests/RegexMasking.js\"") - -} - func TestTapIgnoredUserAgents(t *testing.T) { if testing.Short() { t.Skip("ignored acceptance test") diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index fa6ce500c..c9efa7308 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -48,7 +48,6 @@ func init() { tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver") tapCmd.Flags().StringSliceP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector") tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces") - tapCmd.Flags().StringSliceP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies") tapCmd.Flags().Bool(configStructs.EnableRedactionTapName, defaultTapConfig.EnableRedaction, "Enables redaction of potentially sensitive request/response headers and body values") tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size") tapCmd.Flags().String(configStructs.InsertionFilterName, defaultTapConfig.InsertionFilter, "Set the insertion filter. Accepts string or a file path.") diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 5c6d87b79..2b26bfd73 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -230,23 +230,8 @@ func getErrorDisplayTextForK8sTapManagerError(err kubernetes.K8sTapManagerError) } func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) { - var compiledRegexSlice []*api.SerializableRegexp - - if config.Config.Tap.PlainTextFilterRegexes != nil && len(config.Config.Tap.PlainTextFilterRegexes) > 0 { - compiledRegexSlice = make([]*api.SerializableRegexp, 0) - for _, regexStr := range config.Config.Tap.PlainTextFilterRegexes { - compiledRegex, err := api.CompileRegexToSerializableRegexp(regexStr) - if err != nil { - return nil, err - } - compiledRegexSlice = append(compiledRegexSlice, compiledRegex) - } - } - return &api.TrafficFilteringOptions{ - PlainTextMaskingRegexes: compiledRegexSlice, - IgnoredUserAgents: config.Config.Tap.IgnoredUserAgents, - EnableRedaction: config.Config.Tap.EnableRedaction, + IgnoredUserAgents: config.Config.Tap.IgnoredUserAgents, }, nil } diff --git a/cli/config/configStructs/tapConfig.go b/cli/config/configStructs/tapConfig.go index ad43d3ab1..79615e0ec 100644 --- a/cli/config/configStructs/tapConfig.go +++ b/cli/config/configStructs/tapConfig.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "os" "regexp" + "strings" "github.com/up9inc/mizu/cli/uiUtils" "github.com/up9inc/mizu/shared" @@ -15,38 +16,43 @@ import ( ) const ( - GuiPortTapName = "gui-port" - NamespacesTapName = "namespaces" - AllNamespacesTapName = "all-namespaces" - PlainTextFilterRegexesTapName = "regex-masking" - EnableRedactionTapName = "redact" - HumanMaxEntriesDBSizeTapName = "max-entries-db-size" - InsertionFilterName = "insertion-filter" - DryRunTapName = "dry-run" - ServiceMeshName = "service-mesh" - TlsName = "tls" - ProfilerName = "profiler" - MaxLiveStreamsName = "max-live-streams" + GuiPortTapName = "gui-port" + NamespacesTapName = "namespaces" + AllNamespacesTapName = "all-namespaces" + EnableRedactionTapName = "redact" + HumanMaxEntriesDBSizeTapName = "max-entries-db-size" + InsertionFilterName = "insertion-filter" + DryRunTapName = "dry-run" + ServiceMeshName = "service-mesh" + TlsName = "tls" + ProfilerName = "profiler" + MaxLiveStreamsName = "max-live-streams" ) type TapConfig struct { - PodRegexStr string `yaml:"regex" default:".*"` - GuiPort uint16 `yaml:"gui-port" default:"8899"` - ProxyHost string `yaml:"proxy-host" default:"127.0.0.1"` - Namespaces []string `yaml:"namespaces"` - AllNamespaces bool `yaml:"all-namespaces" default:"false"` - PlainTextFilterRegexes []string `yaml:"regex-masking"` - IgnoredUserAgents []string `yaml:"ignored-user-agents"` - EnableRedaction bool `yaml:"redact" default:"false"` - HumanMaxEntriesDBSize string `yaml:"max-entries-db-size" default:"200MB"` - InsertionFilter string `yaml:"insertion-filter" default:""` - DryRun bool `yaml:"dry-run" default:"false"` - ApiServerResources shared.Resources `yaml:"api-server-resources"` - TapperResources shared.Resources `yaml:"tapper-resources"` - ServiceMesh bool `yaml:"service-mesh" default:"false"` - Tls bool `yaml:"tls" default:"false"` - Profiler bool `yaml:"profiler" default:"false"` - MaxLiveStreams int `yaml:"max-live-streams" default:"500"` + PodRegexStr string `yaml:"regex" default:".*"` + GuiPort uint16 `yaml:"gui-port" default:"8899"` + ProxyHost string `yaml:"proxy-host" default:"127.0.0.1"` + Namespaces []string `yaml:"namespaces"` + AllNamespaces bool `yaml:"all-namespaces" default:"false"` + IgnoredUserAgents []string `yaml:"ignored-user-agents"` + EnableRedaction bool `yaml:"redact" default:"false"` + RedactPatterns struct { + RequestHeaders []string `yaml:"request-headers"` + ResponseHeaders []string `yaml:"response-headers"` + RequestBody []string `yaml:"request-body"` + ResponseBody []string `yaml:"response-body"` + RequestQueryParams []string `yaml:"request-query-params"` + } `yaml:"redact-patterns"` + HumanMaxEntriesDBSize string `yaml:"max-entries-db-size" default:"200MB"` + InsertionFilter string `yaml:"insertion-filter" default:""` + DryRun bool `yaml:"dry-run" default:"false"` + ApiServerResources shared.Resources `yaml:"api-server-resources"` + TapperResources shared.Resources `yaml:"tapper-resources"` + ServiceMesh bool `yaml:"service-mesh" default:"false"` + Tls bool `yaml:"tls" default:"false"` + Profiler bool `yaml:"profiler" default:"false"` + MaxLiveStreams int `yaml:"max-live-streams" default:"500"` } func (config *TapConfig) PodRegex() *regexp.Regexp { @@ -71,9 +77,48 @@ func (config *TapConfig) GetInsertionFilter() string { } } } + + redactFilter := getRedactFilter(config) + if insertionFilter != "" && redactFilter != "" { + return fmt.Sprintf("(%s) and (%s)", insertionFilter, redactFilter) + } else if insertionFilter == "" && redactFilter != "" { + return redactFilter + } + return insertionFilter } +func getRedactFilter(config *TapConfig) string { + if !config.EnableRedaction { + return "" + } + + var redactValues []string + for _, requestHeader := range config.RedactPatterns.RequestHeaders { + redactValues = append(redactValues, fmt.Sprintf("request.headers['%s']", requestHeader)) + } + for _, responseHeader := range config.RedactPatterns.ResponseHeaders { + redactValues = append(redactValues, fmt.Sprintf("response.headers['%s']", responseHeader)) + } + + for _, requestBody := range config.RedactPatterns.RequestBody { + redactValues = append(redactValues, fmt.Sprintf("request.postData.text.json()...%s", requestBody)) + } + for _, responseBody := range config.RedactPatterns.ResponseBody { + redactValues = append(redactValues, fmt.Sprintf("response.content.text.json()...%s", responseBody)) + } + + for _, requestQueryParams := range config.RedactPatterns.RequestQueryParams { + redactValues = append(redactValues, fmt.Sprintf("request.queryString['%s']", requestQueryParams)) + } + + if len(redactValues) == 0 { + return "" + } + + return fmt.Sprintf("redact(\"%s\")", strings.Join(redactValues, "\",\"")) +} + func (config *TapConfig) Validate() error { _, compileErr := regexp.Compile(config.PodRegexStr) if compileErr != nil { diff --git a/performance_analysis/run_tapper_benchmark.sh b/performance_analysis/run_tapper_benchmark.sh index d2b7e6ccd..bebcd7825 100755 --- a/performance_analysis/run_tapper_benchmark.sh +++ b/performance_analysis/run_tapper_benchmark.sh @@ -57,7 +57,7 @@ log "Writing output to $MIZU_BENCHMARK_OUTPUT_DIR" cd $MIZU_HOME || exit 1 export HOST_MODE=0 -export SENSITIVE_DATA_FILTERING_OPTIONS='{"EnableRedaction": false}' +export SENSITIVE_DATA_FILTERING_OPTIONS='{}' export MIZU_DEBUG_DISABLE_PCAP=false export MIZU_DEBUG_DISABLE_TCP_REASSEMBLY=false export MIZU_DEBUG_DISABLE_TCP_STREAM=false diff --git a/tap/api/options.go b/tap/api/options.go index 78c9ad095..1f2ce2579 100644 --- a/tap/api/options.go +++ b/tap/api/options.go @@ -1,7 +1,5 @@ package api type TrafficFilteringOptions struct { - IgnoredUserAgents []string - PlainTextMaskingRegexes []*SerializableRegexp - EnableRedaction bool + IgnoredUserAgents []string } diff --git a/tap/extensions/http/handlers.go b/tap/extensions/http/handlers.go index 870cf511a..4560e4a5b 100644 --- a/tap/extensions/http/handlers.go +++ b/tap/extensions/http/handlers.go @@ -18,10 +18,6 @@ func filterAndEmit(item *api.OutputChannelItem, emitter api.Emitter, options *ap return } - if options.EnableRedaction { - FilterSensitiveData(item, options) - } - replaceForwardedFor(item) emitter.Emit(item) diff --git a/tap/extensions/http/sensitive_data_cleaner.go b/tap/extensions/http/sensitive_data_cleaner.go index 17b0fb435..0e8885f8e 100644 --- a/tap/extensions/http/sensitive_data_cleaner.go +++ b/tap/extensions/http/sensitive_data_cleaner.go @@ -1,30 +1,14 @@ package http import ( - "bytes" - "encoding/json" - "encoding/xml" - "errors" - "fmt" - "io/ioutil" "net/http" - "net/url" "strings" - "github.com/beevik/etree" "github.com/up9inc/mizu/tap/api" ) -const maskedFieldPlaceholderValue = "[REDACTED]" const userAgent = "user-agent" -//these values MUST be all lower case and contain no `-` or `_` characters -var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password", - "username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt", - "bearer", "clientid", "clientsecret", "redirecturi", "phonenumber", - "zip", "zipcode", "address", "country", "firstname", "lastname", - "middlename", "fname", "lname", "birthdate"} - func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) bool { if item.Protocol.Name != "http" { return false @@ -48,192 +32,3 @@ func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteri return false } - -func FilterSensitiveData(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) { - request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request) - response := item.Pair.Response.Payload.(HTTPPayload).Data.(*http.Response) - - filterHeaders(&request.Header) - filterHeaders(&response.Header) - filterUrl(request.URL) - filterRequestBody(request, options) - filterResponseBody(response, options) -} - -func filterRequestBody(request *http.Request, options *api.TrafficFilteringOptions) { - contenType := getContentTypeHeaderValue(request.Header) - body, err := ioutil.ReadAll(request.Body) - if err != nil { - return - } - filteredBody, err := filterHttpBody(body, contenType, options) - if err == nil { - request.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody)) - } else { - request.Body = ioutil.NopCloser(bytes.NewBuffer(body)) - } -} - -func filterResponseBody(response *http.Response, options *api.TrafficFilteringOptions) { - contentType := getContentTypeHeaderValue(response.Header) - body, err := ioutil.ReadAll(response.Body) - if err != nil { - return - } - filteredBody, err := filterHttpBody(body, contentType, options) - if err == nil { - response.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody)) - } else { - response.Body = ioutil.NopCloser(bytes.NewBuffer(body)) - } -} - -func filterHeaders(headers *http.Header) { - for key := range *headers { - if strings.ToLower(key) == userAgent { - continue - } - - if strings.ToLower(key) == "cookie" { - headers.Del(key) - } else if isFieldNameSensitive(key) { - headers.Set(key, maskedFieldPlaceholderValue) - } - } -} - -func getContentTypeHeaderValue(headers http.Header) string { - for key := range headers { - if strings.ToLower(key) == "content-type" { - return headers.Get(key) - } - } - return "" -} - -func isFieldNameSensitive(fieldName string) bool { - if fieldName == ":authority" { - return false - } - - name := strings.ToLower(fieldName) - name = strings.ReplaceAll(name, "_", "") - name = strings.ReplaceAll(name, "-", "") - name = strings.ReplaceAll(name, " ", "") - - for _, sensitiveField := range personallyIdentifiableDataFields { - if strings.Contains(name, sensitiveField) { - return true - } - } - - return false -} - -func filterHttpBody(bytes []byte, contentType string, options *api.TrafficFilteringOptions) ([]byte, error) { - mimeType := strings.Split(contentType, ";")[0] - switch strings.ToLower(mimeType) { - case "application/json": - return filterJsonBody(bytes) - case "text/html": - fallthrough - case "application/xhtml+xml": - fallthrough - case "text/xml": - fallthrough - case "application/xml": - return filterXmlEtree(bytes) - case "text/plain": - if options != nil && options.PlainTextMaskingRegexes != nil { - return filterPlainText(bytes, options), nil - } - } - return bytes, nil -} - -func filterPlainText(bytes []byte, options *api.TrafficFilteringOptions) []byte { - for _, regex := range options.PlainTextMaskingRegexes { - bytes = regex.ReplaceAll(bytes, []byte(maskedFieldPlaceholderValue)) - } - return bytes -} - -func filterXmlEtree(bytes []byte) ([]byte, error) { - if !IsValidXML(bytes) { - return nil, errors.New("Invalid XML") - } - xmlDoc := etree.NewDocument() - err := xmlDoc.ReadFromBytes(bytes) - if err != nil { - return nil, err - } else { - filterXmlElement(xmlDoc.Root()) - } - return xmlDoc.WriteToBytes() -} - -func IsValidXML(data []byte) bool { - return xml.Unmarshal(data, new(interface{})) == nil -} - -func filterXmlElement(element *etree.Element) { - for i, attribute := range element.Attr { - if isFieldNameSensitive(attribute.Key) { - element.Attr[i].Value = maskedFieldPlaceholderValue - } - } - if element.ChildElements() == nil || len(element.ChildElements()) == 0 { - if isFieldNameSensitive(element.Tag) { - element.SetText(maskedFieldPlaceholderValue) - } - } else { - for _, element := range element.ChildElements() { - filterXmlElement(element) - } - } -} - -func filterJsonBody(bytes []byte) ([]byte, error) { - var bodyJsonMap map[string]interface{} - err := json.Unmarshal(bytes, &bodyJsonMap) - if err != nil { - return nil, err - } - filterJsonMap(bodyJsonMap) - return json.Marshal(bodyJsonMap) -} - -func filterJsonMap(jsonMap map[string]interface{}) { - for key, value := range jsonMap { - // Do not replace nil values with maskedFieldPlaceholderValue - if value == nil { - continue - } - - nestedMap, isNested := value.(map[string]interface{}) - if isNested { - filterJsonMap(nestedMap) - } else { - if isFieldNameSensitive(key) { - jsonMap[key] = maskedFieldPlaceholderValue - } - } - } -} - -func filterUrl(url *url.URL) { - if len(url.RawQuery) > 0 { - newQueryArgs := make([]string, 0) - for urlQueryParamName, urlQueryParamValues := range url.Query() { - newValues := urlQueryParamValues - if isFieldNameSensitive(urlQueryParamName) { - newValues = []string{maskedFieldPlaceholderValue} - } - for _, paramValue := range newValues { - newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, paramValue)) - } - } - - url.RawQuery = strings.Join(newQueryArgs, "&") - } -}