From 19fba89ca5d8d830c3c3ab0bc31fb52818b96c02 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Sun, 30 May 2021 15:21:39 +0300 Subject: [PATCH] Update main.go, consts.go, and 2 more files... --- api/main.go | 4 +- api/pkg/utils/consts.go | 2 +- api/pkg/utils/messageSensitiveDataCleaner.go | 109 +++++++++++++++++++ api/pkg/utils/utils.go | 85 --------------- 4 files changed, 113 insertions(+), 87 deletions(-) create mode 100644 api/pkg/utils/messageSensitiveDataCleaner.go diff --git a/api/main.go b/api/main.go index 5ed4f395f..e43872afc 100644 --- a/api/main.go +++ b/api/main.go @@ -34,7 +34,9 @@ func main() { if *standalone { harOutputChannel := tap.StartPassiveTapper() - go api.StartReadingEntries(harOutputChannel, tap.HarOutputDir) + filteredHarChannel := make(chan *tap.OutputChannelItem) + go filterHarHeaders(harOutputChannel, filteredHarChannel) + go api.StartReadingEntries(filteredHarChannel, nil) hostApi(nil) } else if *shouldTap { if *aggregatorAddress == "" { diff --git a/api/pkg/utils/consts.go b/api/pkg/utils/consts.go index 377920a51..0b1a99f3d 100644 --- a/api/pkg/utils/consts.go +++ b/api/pkg/utils/consts.go @@ -6,4 +6,4 @@ var personallyIdentifiableDataFields = []string {"token", "authorization", "auth "bearer", "clientid", "clientsecret", "redirecturi", "phonenumber", "zip", "zipcode", "address", "country", "city", "state", "residence", "name", "firstname", "lastname", "suffix", "middlename", "fname", "lname", - "mname", "date", "birthday", "birthday", "bday", "sender", "receiver"} + "mname", "birthday", "birthday", "birthdate", "bday", "sender", "receiver"} diff --git a/api/pkg/utils/messageSensitiveDataCleaner.go b/api/pkg/utils/messageSensitiveDataCleaner.go new file mode 100644 index 000000000..595751ff7 --- /dev/null +++ b/api/pkg/utils/messageSensitiveDataCleaner.go @@ -0,0 +1,109 @@ +package utils + +import ( + "encoding/json" + "fmt" + "github.com/google/martian/har" + "mizuserver/pkg/tap" + "net/url" + "strings" + +) + +func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem) { + filterHarHeaders(harOutputItem.HarEntry.Request.Headers) + filterHarHeaders(harOutputItem.HarEntry.Response.Headers) + + harOutputItem.HarEntry.Request.URL = filterUrl(harOutputItem.HarEntry.Request.URL) + for i, queryString := range harOutputItem.HarEntry.Request.QueryString { + if isFieldNameSensitive(queryString.Name) { + harOutputItem.HarEntry.Request.QueryString[i].Value = maskedFieldPlaceholderValue + } + } + + if harOutputItem.HarEntry.Request.PostData != nil { + filteredRequestBody, err := filterHttpBody([]byte(harOutputItem.HarEntry.Request.PostData.Text)) + if err == nil { + harOutputItem.HarEntry.Request.PostData.Text = string(filteredRequestBody) + } + } + if harOutputItem.HarEntry.Response.Content != nil { + filteredResponseBody, err := filterHttpBody(harOutputItem.HarEntry.Response.Content.Text) + if err == nil { + harOutputItem.HarEntry.Response.Content.Text = filteredResponseBody + } + } +} + +func filterHarHeaders(headers []har.Header) { + for i, header := range headers { + if isFieldNameSensitive(header.Name) { + headers[i].Value = maskedFieldPlaceholderValue + } + } +} + +func isFieldNameSensitive(fieldName string) bool { + name := strings.ToLower(fieldName) + name = strings.ReplaceAll(name, "_", "") + name = strings.ReplaceAll(name, "-", "") + name = strings.ReplaceAll(name, " ", "") + + for _, sensitiveField := range personallyIdentifiableDataFields { + if strings.Contains(name, sensitiveField) { + return true + } + } + + return false +} + +func filterHttpBody(bytes []byte) ([]byte, error){ + var bodyJsonMap map[string] interface{} + err := json.Unmarshal(bytes ,&bodyJsonMap) + if err != nil { + return nil, err + } + filterJsonMap(bodyJsonMap) + return json.Marshal(bodyJsonMap) +} + +func filterJsonMap(jsonMap map[string] interface{}) { + for key, value := range jsonMap { + if value == nil { + return + } + nestedMap, isNested := value.(map[string] interface{}) + if isNested { + filterJsonMap(nestedMap) + } else { + if isFieldNameSensitive(key) { + jsonMap[key] = maskedFieldPlaceholderValue + } + } + } +} + +func filterUrl(originalUrl string) string { + parsedUrl, err := url.Parse(originalUrl) + if err != nil { + return originalUrl + } else { + if len(parsedUrl.RawQuery) > 0 { + newQueryArgs := make([]string, 0) + for urlQueryParamName, urlQueryParamValues := range parsedUrl.Query() { + newValues := urlQueryParamValues + if isFieldNameSensitive(urlQueryParamName) { + newValues = []string {maskedFieldPlaceholderValue} + } + for _, paramValue := range newValues { + newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, paramValue)) + } + } + + parsedUrl.RawQuery = strings.Join(newQueryArgs, "&") + } + + return parsedUrl.String() + } +} diff --git a/api/pkg/utils/utils.go b/api/pkg/utils/utils.go index 8b2638fb1..ebf4ce9f1 100644 --- a/api/pkg/utils/utils.go +++ b/api/pkg/utils/utils.go @@ -4,15 +4,12 @@ import ( "encoding/json" "fmt" "github.com/gofiber/fiber/v2" - "github.com/google/martian/har" "log" "mizuserver/pkg/models" - "mizuserver/pkg/tap" "net/url" "os" "os/signal" "reflect" - "strings" "syscall" ) @@ -88,85 +85,3 @@ func GetBytesFromStruct(v interface{}) []byte{ a, _ := json.Marshal(v) return a } - -func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem) { - filterHarHeaders(harOutputItem.HarEntry.Request.Headers) - filterHarHeaders(harOutputItem.HarEntry.Response.Headers) - - harOutputItem.HarEntry.Request.URL = filterUrl(harOutputItem.HarEntry.Request.URL) - - var requestJsonMap map[string] interface{} - err := json.Unmarshal([]byte(harOutputItem.HarEntry.Request.PostData.Text) ,&requestJsonMap) - if err == nil { - filterJsonMap(requestJsonMap) - } - // - //filterJsonMap(harOutputItem.HarEntry.Response.Content.Text) - - - // filter url query params - // filter bodies -} - -func filterHarHeaders(headers []har.Header) { - for _, header := range headers { - if isFieldNameSensitive(header.Name) { - header.Value = maskedFieldPlaceholderValue - } - } -} - -func isFieldNameSensitive(fieldName string) bool { - name := strings.ToLower(fieldName) - name = strings.ReplaceAll(name, "_", "") - name = strings.ReplaceAll(name, "-", "") - name = strings.ReplaceAll(name, " ", "") - - for _, sensitiveField := range personallyIdentifiableDataFields { - if strings.Contains(name, sensitiveField) { - return true - } - } - - return false -} - -func filterJsonMap(jsonMap map[string] interface{}) { - for key, value := range jsonMap { - if value == nil { - return - } - nestedMap, isNested := value.(map[string] interface{}) - if isNested { - filterJsonMap(nestedMap) - } else { - if isFieldNameSensitive(key) { - jsonMap[key] = maskedFieldPlaceholderValue - } - } - } -} - -func filterUrl(originalUrl string) string { - parsedUrl, err := url.Parse(originalUrl) - if err != nil { - return originalUrl - } else { - if len(parsedUrl.RawQuery) > 0 { - newQueryArgs := make([]string, 0) - for urlQueryParamName, urlQueryParamValues := range parsedUrl.Query() { - newValues := urlQueryParamValues - if isFieldNameSensitive(urlQueryParamName) { - newValues = []string {maskedFieldPlaceholderValue} - } - for value := range newValues { - newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, value)) - } - } - - parsedUrl.RawQuery = strings.Join(newQueryArgs, "&") - } - - return parsedUrl.String() - } -}