Update main.go, consts.go, and 2 more files...

api/main.go

@@ -34,7 +34,9 @@ func main() {
 
 	if *standalone {
 		harOutputChannel := tap.StartPassiveTapper()
-		go api.StartReadingEntries(harOutputChannel, tap.HarOutputDir)
+		filteredHarChannel := make(chan *tap.OutputChannelItem)
+		go filterHarHeaders(harOutputChannel, filteredHarChannel)
+		go api.StartReadingEntries(filteredHarChannel, nil)
 		hostApi(nil)
 	} else if *shouldTap {
 		if *aggregatorAddress == "" {

api/pkg/utils/consts.go

@@ -6,4 +6,4 @@ var personallyIdentifiableDataFields = []string {"token", "authorization", "auth
 												 "bearer", "clientid", "clientsecret", "redirecturi", "phonenumber",
 												 "zip", "zipcode", "address", "country", "city", "state", "residence",
                                                  "name", "firstname", "lastname", "suffix", "middlename", "fname", "lname",
-                                                 "mname", "date", "birthday", "birthday", "bday", "sender", "receiver"}
+                                                 "mname", "birthday", "birthday", "birthdate", "bday", "sender", "receiver"}

api/pkg/utils/messageSensitiveDataCleaner.go

@@ -0,0 +1,109 @@
+package utils
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/google/martian/har"
+	"mizuserver/pkg/tap"
+	"net/url"
+	"strings"
+
+)
+
+func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem) {
+	filterHarHeaders(harOutputItem.HarEntry.Request.Headers)
+	filterHarHeaders(harOutputItem.HarEntry.Response.Headers)
+
+	harOutputItem.HarEntry.Request.URL = filterUrl(harOutputItem.HarEntry.Request.URL)
+	for i, queryString := range harOutputItem.HarEntry.Request.QueryString {
+		if isFieldNameSensitive(queryString.Name) {
+			harOutputItem.HarEntry.Request.QueryString[i].Value = maskedFieldPlaceholderValue
+		}
+	}
+
+	if harOutputItem.HarEntry.Request.PostData != nil {
+		filteredRequestBody, err := filterHttpBody([]byte(harOutputItem.HarEntry.Request.PostData.Text))
+		if err == nil {
+			harOutputItem.HarEntry.Request.PostData.Text = string(filteredRequestBody)
+		}
+	}
+	if harOutputItem.HarEntry.Response.Content != nil {
+		filteredResponseBody, err := filterHttpBody(harOutputItem.HarEntry.Response.Content.Text)
+		if err == nil {
+			harOutputItem.HarEntry.Response.Content.Text = filteredResponseBody
+		}
+	}
+}
+
+func filterHarHeaders(headers []har.Header) {
+	for i, header := range headers {
+		if isFieldNameSensitive(header.Name) {
+			headers[i].Value = maskedFieldPlaceholderValue
+		}
+	}
+}
+
+func isFieldNameSensitive(fieldName string) bool {
+	name := strings.ToLower(fieldName)
+	name = strings.ReplaceAll(name, "_", "")
+	name = strings.ReplaceAll(name, "-", "")
+	name = strings.ReplaceAll(name, " ", "")
+
+	for _, sensitiveField := range personallyIdentifiableDataFields {
+		if strings.Contains(name, sensitiveField) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func filterHttpBody(bytes []byte) ([]byte, error){
+	var bodyJsonMap map[string] interface{}
+	err := json.Unmarshal(bytes ,&bodyJsonMap)
+	if err != nil {
+		return nil, err
+	}
+	filterJsonMap(bodyJsonMap)
+	return json.Marshal(bodyJsonMap)
+}
+
+func filterJsonMap(jsonMap map[string] interface{}) {
+	for key, value := range jsonMap {
+		if value == nil {
+			return
+		}
+		nestedMap, isNested := value.(map[string] interface{})
+		if isNested {
+			filterJsonMap(nestedMap)
+		} else {
+			if isFieldNameSensitive(key) {
+				jsonMap[key] = maskedFieldPlaceholderValue
+			}
+		}
+	}
+}
+
+func filterUrl(originalUrl string) string {
+	parsedUrl, err := url.Parse(originalUrl)
+	if err != nil {
+		return originalUrl
+	} else {
+		if len(parsedUrl.RawQuery) > 0 {
+			newQueryArgs := make([]string, 0)
+			for urlQueryParamName, urlQueryParamValues := range parsedUrl.Query() {
+				newValues := urlQueryParamValues
+				if isFieldNameSensitive(urlQueryParamName) {
+					newValues = []string {maskedFieldPlaceholderValue}
+				}
+				for _, paramValue := range newValues {
+					newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, paramValue))
+				}
+			}
+
+			parsedUrl.RawQuery = strings.Join(newQueryArgs, "&")
+		}
+
+		return parsedUrl.String()
+	}
+}

api/pkg/utils/utils.go

@@ -4,15 +4,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/gofiber/fiber/v2"
-	"github.com/google/martian/har"
 	"log"
 	"mizuserver/pkg/models"
-	"mizuserver/pkg/tap"
 	"net/url"
 	"os"
 	"os/signal"
 	"reflect"
-	"strings"
 	"syscall"
 )
 
@@ -88,85 +85,3 @@ func GetBytesFromStruct(v interface{}) []byte{
 	a, _ := json.Marshal(v)
 	return a
 }
-
-func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem) {
-	filterHarHeaders(harOutputItem.HarEntry.Request.Headers)
-	filterHarHeaders(harOutputItem.HarEntry.Response.Headers)
-
-	harOutputItem.HarEntry.Request.URL = filterUrl(harOutputItem.HarEntry.Request.URL)
-
-	var requestJsonMap map[string] interface{}
-	err := json.Unmarshal([]byte(harOutputItem.HarEntry.Request.PostData.Text) ,&requestJsonMap)
-	if err == nil {
-		filterJsonMap(requestJsonMap)
-	}
-	//
-	//filterJsonMap(harOutputItem.HarEntry.Response.Content.Text)
-
-
-	// filter url query params
-	// filter bodies
-}
-
-func filterHarHeaders(headers []har.Header) {
-	for _, header := range headers {
-		if isFieldNameSensitive(header.Name) {
-			header.Value = maskedFieldPlaceholderValue
-		}
-	}
-}
-
-func isFieldNameSensitive(fieldName string) bool {
-	name := strings.ToLower(fieldName)
-	name = strings.ReplaceAll(name, "_", "")
-	name = strings.ReplaceAll(name, "-", "")
-	name = strings.ReplaceAll(name, " ", "")
-
-	for _, sensitiveField := range personallyIdentifiableDataFields {
-		if strings.Contains(name, sensitiveField) {
-			return true
-		}
-	}
-
-	return false
-}
-
-func filterJsonMap(jsonMap map[string] interface{}) {
-	for key, value := range jsonMap {
-		if value == nil {
-			return
-		}
-		nestedMap, isNested := value.(map[string] interface{})
-		if isNested {
-			filterJsonMap(nestedMap)
-		} else {
-			if isFieldNameSensitive(key) {
-				jsonMap[key] = maskedFieldPlaceholderValue
-			}
-		}
-	}
-}
-
-func filterUrl(originalUrl string) string {
-	parsedUrl, err := url.Parse(originalUrl)
-	if err != nil {
-		return originalUrl
-	} else {
-		if len(parsedUrl.RawQuery) > 0 {
-			newQueryArgs := make([]string, 0)
-			for urlQueryParamName, urlQueryParamValues := range parsedUrl.Query() {
-				newValues := urlQueryParamValues
-				if isFieldNameSensitive(urlQueryParamName) {
-					newValues = []string {maskedFieldPlaceholderValue}
-				}
-				for value := range newValues {
-					newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, value))
-				}
-			}
-
-			parsedUrl.RawQuery = strings.Join(newQueryArgs, "&")
-		}
-
-		return parsedUrl.String()
-	}
-}