diff --git a/agent/pkg/har/utils.go b/agent/pkg/har/utils.go index 3c1289858..cbeafd636 100644 --- a/agent/pkg/har/utils.go +++ b/agent/pkg/har/utils.go @@ -11,75 +11,30 @@ import ( "github.com/up9inc/mizu/logger" ) -// Keep it because we might want cookies in the future -//func BuildCookies(rawCookies []interface{}) []har.Cookie { -// cookies := make([]har.Cookie, 0, len(rawCookies)) -// -// for _, cookie := range rawCookies { -// c := cookie.(map[string]interface{}) -// expiresStr := "" -// if c["expires"] != nil { -// expiresStr = c["expires"].(string) -// } -// expires, _ := time.Parse(time.RFC3339, expiresStr) -// httpOnly := false -// if c["httponly"] != nil { -// httpOnly, _ = strconv.ParseBool(c["httponly"].(string)) -// } -// secure := false -// if c["secure"] != nil { -// secure, _ = strconv.ParseBool(c["secure"].(string)) -// } -// path := "" -// if c["path"] != nil { -// path = c["path"].(string) -// } -// domain := "" -// if c["domain"] != nil { -// domain = c["domain"].(string) -// } -// -// cookies = append(cookies, har.Cookie{ -// Name: c["name"].(string), -// Value: c["value"].(string), -// Path: path, -// Domain: domain, -// HTTPOnly: httpOnly, -// Secure: secure, -// Expires: expires, -// Expires8601: expiresStr, -// }) -// } -// -// return cookies -//} - -func BuildHeaders(rawHeaders []interface{}) ([]Header, string, string, string, string, string) { +func BuildHeaders(rawHeaders map[string]interface{}) ([]Header, string, string, string, string, string) { var host, scheme, authority, path, status string headers := make([]Header, 0, len(rawHeaders)) - for _, header := range rawHeaders { - h := header.(map[string]interface{}) - + for key, value := range rawHeaders { headers = append(headers, Header{ - Name: h["name"].(string), - Value: h["value"].(string), + Name: key, + Value: value.(string), }) - if h["name"] == "Host" { - host = h["value"].(string) + if key == "Host" { + host = value.(string) } - if h["name"] == ":authority" { - authority = h["value"].(string) + if key == ":authority" { + authority = value.(string) } - if h["name"] == ":scheme" { - scheme = h["value"].(string) + if key == ":scheme" { + scheme = value.(string) } - if h["name"] == ":path" { - path = h["value"].(string) + if key == ":path" { + path = value.(string) } - if h["name"] == ":status" { - status = h["value"].(string) + if key == ":status" { + status = value.(string) } } @@ -119,8 +74,8 @@ func BuildPostParams(rawParams []interface{}) []Param { } func NewRequest(request map[string]interface{}) (harRequest *Request, err error) { - headers, host, scheme, authority, path, _ := BuildHeaders(request["_headers"].([]interface{})) - cookies := make([]Cookie, 0) // BuildCookies(request["_cookies"].([]interface{})) + headers, host, scheme, authority, path, _ := BuildHeaders(request["headers"].(map[string]interface{})) + cookies := make([]Cookie, 0) postData, _ := request["postData"].(map[string]interface{}) mimeType := postData["mimeType"] @@ -134,12 +89,20 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error) } queryString := make([]QueryString, 0) - for _, _qs := range request["_queryString"].([]interface{}) { - qs := _qs.(map[string]interface{}) - queryString = append(queryString, QueryString{ - Name: qs["name"].(string), - Value: qs["value"].(string), - }) + for key, value := range request["queryString"].(map[string]interface{}) { + if valuesInterface, ok := value.([]interface{}); ok { + for _, valueInterface := range valuesInterface { + queryString = append(queryString, QueryString{ + Name: key, + Value: valueInterface.(string), + }) + } + } else { + queryString = append(queryString, QueryString{ + Name: key, + Value: value.(string), + }) + } } url := fmt.Sprintf("http://%s%s", host, request["url"].(string)) @@ -172,8 +135,8 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error) } func NewResponse(response map[string]interface{}) (harResponse *Response, err error) { - headers, _, _, _, _, _status := BuildHeaders(response["_headers"].([]interface{})) - cookies := make([]Cookie, 0) // BuildCookies(response["_cookies"].([]interface{})) + headers, _, _, _, _, _status := BuildHeaders(response["headers"].(map[string]interface{})) + cookies := make([]Cookie, 0) content, _ := response["content"].(map[string]interface{}) mimeType := content["mimeType"] diff --git a/tap/extensions/http/Makefile b/tap/extensions/http/Makefile index 8394dbdba..1d55a9635 100644 --- a/tap/extensions/http/Makefile +++ b/tap/extensions/http/Makefile @@ -13,4 +13,4 @@ test-pull-bin: test-pull-expect: @mkdir -p expect - @[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5' -m cp -r gs://static.up9.io/mizu/test-pcap/expect15/http/\* expect + @[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5' -m cp -r gs://static.up9.io/mizu/test-pcap/expect16/http/\* expect diff --git a/tap/extensions/http/helpers.go b/tap/extensions/http/helpers.go index a8c4b78d7..40f9c913a 100644 --- a/tap/extensions/http/helpers.go +++ b/tap/extensions/http/helpers.go @@ -6,13 +6,16 @@ import ( "reflect" "sort" "strconv" + "strings" "github.com/up9inc/mizu/tap/api" ) func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}) { newMap = make(map[string]interface{}) - for _, item := range mapSlice { + + mergedMapSlice := mapSliceMergeRepeatedKeys(mapSlice) + for _, item := range mergedMapSlice { h := item.(map[string]interface{}) newMap[h["name"].(string)] = h["value"] } @@ -20,6 +23,28 @@ func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{} return } +func mapSliceRebuildAsMergedMap(mapSlice []interface{}) (newMap map[string]interface{}) { + newMap = make(map[string]interface{}) + + mergedMapSlice := mapSliceMergeRepeatedKeys(mapSlice) + for _, item := range mergedMapSlice { + h := item.(map[string]interface{}) + + if valuesInterface, ok := h["value"].([]interface{}); ok { + var values []string + for _, valueInterface := range valuesInterface { + values = append(values, valueInterface.(string)) + } + + newMap[h["name"].(string)] = strings.Join(values, ",") + } else { + newMap[h["name"].(string)] = h["value"] + } + } + + return +} + func mapSliceMergeRepeatedKeys(mapSlice []interface{}) (newMapSlice []interface{}) { newMapSlice = make([]interface{}, 0) valuesMap := make(map[string][]interface{}) @@ -47,6 +72,24 @@ func mapSliceMergeRepeatedKeys(mapSlice []interface{}) (newMapSlice []interface{ return } +func representMapAsTable(mapToTable map[string]interface{}, selectorPrefix string) (representation string) { + var table []api.TableData + + keys := make([]string, 0, len(mapToTable)) + for k := range mapToTable { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, key := range keys { + table = append(table, createTableForKey(key, mapToTable[key], selectorPrefix)...) + } + + obj, _ := json.Marshal(table) + representation = string(obj) + return +} + func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (representation string) { var table []api.TableData for _, item := range mapSlice { @@ -54,34 +97,7 @@ func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (re key := h["name"].(string) value := h["value"] - var reflectKind reflect.Kind - reflectType := reflect.TypeOf(value) - if reflectType == nil { - reflectKind = reflect.Interface - } else { - reflectKind = reflect.TypeOf(value).Kind() - } - - switch reflectKind { - case reflect.Slice: - fallthrough - case reflect.Array: - for i, el := range value.([]interface{}) { - selector := fmt.Sprintf("%s.%s[%d]", selectorPrefix, key, i) - table = append(table, api.TableData{ - Name: fmt.Sprintf("%s [%d]", key, i), - Value: el, - Selector: selector, - }) - } - default: - selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, key) - table = append(table, api.TableData{ - Name: key, - Value: value, - Selector: selector, - }) - } + table = append(table, createTableForKey(key, value, selectorPrefix)...) } obj, _ := json.Marshal(table) @@ -89,6 +105,41 @@ func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (re return } +func createTableForKey(key string, value interface{}, selectorPrefix string) []api.TableData { + var table []api.TableData + + var reflectKind reflect.Kind + reflectType := reflect.TypeOf(value) + if reflectType == nil { + reflectKind = reflect.Interface + } else { + reflectKind = reflect.TypeOf(value).Kind() + } + + switch reflectKind { + case reflect.Slice: + fallthrough + case reflect.Array: + for i, el := range value.([]interface{}) { + selector := fmt.Sprintf("%s.%s[%d]", selectorPrefix, key, i) + table = append(table, api.TableData{ + Name: fmt.Sprintf("%s [%d]", key, i), + Value: el, + Selector: selector, + }) + } + default: + selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, key) + table = append(table, api.TableData{ + Name: key, + Value: value, + Selector: selector, + }) + } + + return table +} + func representSliceAsTable(slice []interface{}, selectorPrefix string) (representation string) { var table []api.TableData for i, item := range slice { diff --git a/tap/extensions/http/main.go b/tap/extensions/http/main.go index 0b42a907d..f871cd8ae 100644 --- a/tap/extensions/http/main.go +++ b/tap/extensions/http/main.go @@ -286,19 +286,13 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, reqDetails["pathSegments"] = strings.Split(path, "/")[1:] // Rearrange the maps for the querying - reqDetails["_headers"] = reqDetails["headers"] - reqDetails["headers"] = mapSliceRebuildAsMap(reqDetails["_headers"].([]interface{})) - resDetails["_headers"] = resDetails["headers"] - resDetails["headers"] = mapSliceRebuildAsMap(resDetails["_headers"].([]interface{})) + reqDetails["headers"] = mapSliceRebuildAsMergedMap(reqDetails["headers"].([]interface{})) + resDetails["headers"] = mapSliceRebuildAsMergedMap(resDetails["headers"].([]interface{})) - reqDetails["_cookies"] = reqDetails["cookies"] - reqDetails["cookies"] = mapSliceRebuildAsMap(reqDetails["_cookies"].([]interface{})) - resDetails["_cookies"] = resDetails["cookies"] - resDetails["cookies"] = mapSliceRebuildAsMap(resDetails["_cookies"].([]interface{})) + reqDetails["cookies"] = mapSliceRebuildAsMergedMap(reqDetails["cookies"].([]interface{})) + resDetails["cookies"] = mapSliceRebuildAsMergedMap(resDetails["cookies"].([]interface{})) - reqDetails["_queryString"] = reqDetails["queryString"] - reqDetails["_queryStringMerged"] = mapSliceMergeRepeatedKeys(reqDetails["_queryString"].([]interface{})) - reqDetails["queryString"] = mapSliceRebuildAsMap(reqDetails["_queryStringMerged"].([]interface{})) + reqDetails["queryString"] = mapSliceRebuildAsMap(reqDetails["queryString"].([]interface{})) elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds() if elapsedTime < 0 { @@ -397,19 +391,19 @@ func representRequest(request map[string]interface{}) (repRequest []interface{}) repRequest = append(repRequest, api.SectionData{ Type: api.TABLE, Title: "Headers", - Data: representMapSliceAsTable(request["_headers"].([]interface{}), `request.headers`), + Data: representMapAsTable(request["headers"].(map[string]interface{}), `request.headers`), }) repRequest = append(repRequest, api.SectionData{ Type: api.TABLE, Title: "Cookies", - Data: representMapSliceAsTable(request["_cookies"].([]interface{}), `request.cookies`), + Data: representMapAsTable(request["cookies"].(map[string]interface{}), `request.cookies`), }) repRequest = append(repRequest, api.SectionData{ Type: api.TABLE, Title: "Query String", - Data: representMapSliceAsTable(request["_queryStringMerged"].([]interface{}), `request.queryString`), + Data: representMapAsTable(request["queryString"].(map[string]interface{}), `request.queryString`), }) postData, _ := request["postData"].(map[string]interface{}) @@ -485,13 +479,13 @@ func representResponse(response map[string]interface{}) (repResponse []interface repResponse = append(repResponse, api.SectionData{ Type: api.TABLE, Title: "Headers", - Data: representMapSliceAsTable(response["_headers"].([]interface{}), `response.headers`), + Data: representMapAsTable(response["headers"].(map[string]interface{}), `response.headers`), }) repResponse = append(repResponse, api.SectionData{ Type: api.TABLE, Title: "Cookies", - Data: representMapSliceAsTable(response["_cookies"].([]interface{}), `response.cookies`), + Data: representMapAsTable(response["cookies"].(map[string]interface{}), `response.cookies`), }) content, _ := response["content"].(map[string]interface{})