diff --git a/agent/pkg/api/main.go b/agent/pkg/api/main.go index 904c9313b..f70b8d3f0 100644 --- a/agent/pkg/api/main.go +++ b/agent/pkg/api/main.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "mizuserver/pkg/har" "mizuserver/pkg/holder" "mizuserver/pkg/providers" "os" @@ -15,11 +16,9 @@ import ( "mizuserver/pkg/servicemap" - "github.com/google/martian/har" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" tapApi "github.com/up9inc/mizu/tap/api" - "mizuserver/pkg/models" "mizuserver/pkg/oas" "mizuserver/pkg/resolver" @@ -132,7 +131,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension mizuEntry.ContractContent = contract.Content } - harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime) + harEntry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime) if err == nil { rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name) mizuEntry.Rules = rules diff --git a/agent/pkg/controllers/entries_controller.go b/agent/pkg/controllers/entries_controller.go index 61a2b2301..12bd2a8e6 100644 --- a/agent/pkg/controllers/entries_controller.go +++ b/agent/pkg/controllers/entries_controller.go @@ -2,8 +2,8 @@ package controllers import ( "encoding/json" + "mizuserver/pkg/har" "mizuserver/pkg/models" - "mizuserver/pkg/utils" "mizuserver/pkg/validation" "net/http" "strconv" @@ -127,7 +127,7 @@ func GetEntry(c *gin.Context) { var rules []map[string]interface{} var isRulesEnabled bool if entry.Protocol.Name == "http" { - harEntry, _ := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime) + harEntry, _ := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime) _, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name) isRulesEnabled = _isRulesEnabled inrec, _ := json.Marshal(rulesMatched) diff --git a/agent/pkg/har/types.go b/agent/pkg/har/types.go new file mode 100644 index 000000000..901560906 --- /dev/null +++ b/agent/pkg/har/types.go @@ -0,0 +1,375 @@ +package har + +import ( + "encoding/base64" + "github.com/up9inc/mizu/shared/logger" + "time" + "unicode/utf8" +) + +/* +HTTP Archive (HAR) format +https://w3c.github.io/web-performance/specs/HAR/Overview.html +*/ + +// HAR is a container type for deserialization +type HAR struct { + Log Log `json:"log"` +} + +// Log represents the root of the exported data. This object MUST be present and its name MUST be "log". +type Log struct { + // The object contains the following name/value pairs: + + // Required. Version number of the format. + Version string `json:"version"` + // Required. An object of type creator that contains the name and version + // information of the log creator application. + Creator Creator `json:"creator"` + // Optional. An object of type browser that contains the name and version + // information of the user agent. + Browser Browser `json:"browser"` + // Optional. An array of objects of type page, each representing one exported + // (tracked) page. Leave out this field if the application does not support + // grouping by pages. + Pages []Page `json:"pages,omitempty"` + // Required. An array of objects of type entry, each representing one + // exported (tracked) HTTP request. + Entries []Entry `json:"entries"` + // Optional. A comment provided by the user or the application. Sorting + // entries by startedDateTime (starting from the oldest) is preferred way how + // to export data since it can make importing faster. However the reader + // application should always make sure the array is sorted (if required for + // the import). + Comment string `json:"comment"` +} + +// Creator contains information about the log creator application +type Creator struct { + // Required. The name of the application that created the log. + Name string `json:"name"` + // Required. The version number of the application that created the log. + Version string `json:"version"` + // Optional. A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// Browser that created the log +type Browser struct { + // Required. The name of the browser that created the log. + Name string `json:"name"` + // Required. The version number of the browser that created the log. + Version string `json:"version"` + // Optional. A comment provided by the user or the browser. + Comment string `json:"comment"` +} + +// Page object for every exported web page and one object for every HTTP request. +// In case when an HTTP trace tool isn't able to group requests by a page, +// the object is empty and individual requests doesn't have a parent page. +type Page struct { + /* There is one object for every exported web page and one + object for every HTTP request. In case when an HTTP trace tool isn't able to + group requests by a page, the object is empty and individual + requests doesn't have a parent page. + */ + + // Date and time stamp for the beginning of the page load + // (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00). + StartedDateTime string `json:"startedDateTime"` + // Unique identifier of a page within the . Entries use it to refer the parent page. + ID string `json:"id"` + // Page title. + Title string `json:"title"` + // Detailed timing info about page load. + PageTiming PageTiming `json:"pageTiming"` + // (new in 1.2) A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// PageTiming describes timings for various events (states) fired during the page load. +// All times are specified in milliseconds. If a time info is not available appropriate field is set to -1. +type PageTiming struct { + // Content of the page loaded. Number of milliseconds since page load started + // (page.startedDateTime). Use -1 if the timing does not apply to the current + // request. + // Depeding on the browser, onContentLoad property represents DOMContentLoad + // event or document.readyState == interactive. + OnContentLoad int `json:"onContentLoad"` + // Page is loaded (onLoad event fired). Number of milliseconds since page + // load started (page.startedDateTime). Use -1 if the timing does not apply + // to the current request. + OnLoad int `json:"onLoad"` + // (new in 1.2) A comment provided by the user or the application. + Comment string `json:"comment"` +} + +// Entry is a unique, optional Reference to the parent page. +// Leave out this field if the application does not support grouping by pages. +type Entry struct { + Pageref string `json:"pageref,omitempty"` + // Date and time stamp of the request start + // (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD). + StartedDateTime string `json:"startedDateTime"` + // Total elapsed time of the request in milliseconds. This is the sum of all + // timings available in the timings object (i.e. not including -1 values) . + Time int `json:"time"` + // Detailed info about the request. + Request Request `json:"request"` + // Detailed info about the response. + Response Response `json:"response"` + // Info about cache usage. + Cache Cache `json:"cache"` + // Detailed timing info about request/response round trip. + PageTimings PageTimings `json:"pageTimings"` + // optional (new in 1.2) IP address of the server that was connected + // (result of DNS resolution). + ServerIPAddress string `json:"serverIPAddress,omitempty"` + // optional (new in 1.2) Unique ID of the parent TCP/IP connection, can be + // the client port number. Note that a port number doesn't have to be unique + // identifier in cases where the port is shared for more connections. If the + // port isn't available for the application, any other unique connection ID + // can be used instead (e.g. connection index). Leave out this field if the + // application doesn't support this info. + Connection string `json:"connection,omitempty"` + // (new in 1.2) A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// Request contains detailed info about performed request. +type Request struct { + // Request method (GET, POST, ...). + Method string `json:"method"` + // Absolute URL of the request (fragments are not included). + URL string `json:"url"` + // Request HTTP Version. + HTTPVersion string `json:"httpVersion"` + // List of cookie objects. + Cookies []Cookie `json:"cookies"` + // List of header objects. + Headers []NVP `json:"headers"` + // List of query parameter objects. + QueryString []NVP `json:"queryString"` + // Posted data. + PostData PostData `json:"postData"` + // Total number of bytes from the start of the HTTP request message until + // (and including) the double CRLF before the body. Set to -1 if the info + // is not available. + HeaderSize int `json:"headerSize"` + // Size of the request body (POST data payload) in bytes. Set to -1 if the + // info is not available. + BodySize int `json:"bodySize"` + // (new in 1.2) A comment provided by the user or the application. + Comment string `json:"comment"` +} + +// Response contains detailed info about the response. +type Response struct { + // Response status. + Status int `json:"status"` + // Response status description. + StatusText string `json:"statusText"` + // Response HTTP Version. + HTTPVersion string `json:"httpVersion"` + // List of cookie objects. + Cookies []Cookie `json:"cookies"` + // List of header objects. + Headers []NVP `json:"headers"` + // Details about the response body. + Content Content `json:"content"` + // Redirection target URL from the Location response header. + RedirectURL string `json:"redirectURL"` + // Total number of bytes from the start of the HTTP response message until + // (and including) the double CRLF before the body. Set to -1 if the info is + // not available. + // The size of received response-headers is computed only from headers that + // are really received from the server. Additional headers appended by the + // browser are not included in this number, but they appear in the list of + // header objects. + HeadersSize int `json:"headersSize"` + // Size of the received response body in bytes. Set to zero in case of + // responses coming from the cache (304). Set to -1 if the info is not + // available. + BodySize int `json:"bodySize"` + // optional (new in 1.2) A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// Cookie contains list of all cookies (used in and objects). +type Cookie struct { + // The name of the cookie. + Name string `json:"name"` + // The cookie value. + Value string `json:"value"` + // optional The path pertaining to the cookie. + Path string `json:"path,omitempty"` + // optional The host of the cookie. + Domain string `json:"domain,omitempty"` + // optional Cookie expiration time. + // (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.123+02:00). + Expires string `json:"expires,omitempty"` + // optional Set to true if the cookie is HTTP only, false otherwise. + HTTPOnly bool `json:"httpOnly,omitempty"` + // optional (new in 1.2) True if the cookie was transmitted over ssl, false + // otherwise. + Secure bool `json:"secure,omitempty"` + // optional (new in 1.2) A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// NVP is simply a name/value pair with a comment +type NVP struct { + Name string `json:"name"` + Value string `json:"value"` + Comment string `json:"comment,omitempty"` +} + +// PostData describes posted data, if any (embedded in object). +type PostData struct { + // Mime type of posted data. + MimeType string `json:"mimeType"` + // List of posted parameters (in case of URL encoded parameters). + Params []PostParam `json:"params"` + // Plain text posted data + Text string `json:"text"` + // optional (new in 1.2) A comment provided by the user or the + // application. + Comment string `json:"comment,omitempty"` +} + +func (d PostData) B64Decoded() (bool, []byte, string) { + // there is a weird gap in HAR spec 1.2, that does not define encoding for binary POST bodies + // we have own convention of putting `base64` into comment field to handle it similar to response `Content` + return b64Decoded(d.Comment, d.Text) +} + +// PostParam is a list of posted parameters, if any (embedded in object). +type PostParam struct { + // name of a posted parameter. + Name string `json:"name"` + // optional value of a posted parameter or content of a posted file. + Value string `json:"value,omitempty"` + // optional name of a posted file. + FileName string `json:"fileName,omitempty"` + // optional content type of a posted file. + ContentType string `json:"contentType,omitempty"` + // optional (new in 1.2) A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// Content describes details about response content (embedded in object). +type Content struct { + // Length of the returned content in bytes. Should be equal to + // response.bodySize if there is no compression and bigger when the content + // has been compressed. + Size int `json:"size"` + // optional Number of bytes saved. Leave out this field if the information + // is not available. + Compression int `json:"compression,omitempty"` + // MIME type of the response text (value of the Content-Type response + // header). The charset attribute of the MIME type is included (if + // available). + MimeType string `json:"mimeType"` + // optional Response body sent from the server or loaded from the browser + // cache. This field is populated with textual content only. The text field + // is either HTTP decoded text or a encoded (e.g. "base64") representation of + // the response body. Leave out this field if the information is not + // available. + Text string `json:"text,omitempty"` + // optional (new in 1.2) Encoding used for response text field e.g + // "base64". Leave out this field if the text field is HTTP decoded + // (decompressed & unchunked), than trans-coded from its original character + // set into UTF-8. + Encoding string `json:"encoding,omitempty"` + // optional (new in 1.2) A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +func (c Content) B64Decoded() (bool, []byte, string) { + return b64Decoded(c.Encoding, c.Text) +} + +func b64Decoded(enc string, text string) (isBinary bool, asBytes []byte, asString string) { + if enc == "base64" { + decoded, err := base64.StdEncoding.DecodeString(text) + if err != nil { + logger.Log.Warningf("Failed to decode content as base64: %s", text) + return false, []byte(text), text + } + valid := utf8.Valid(decoded) + return !valid, decoded, string(decoded) + } else { + return false, nil, text + } +} + +// Cache contains info about a request coming from browser cache. +type Cache struct { + // optional State of a cache entry before the request. Leave out this field + // if the information is not available. + BeforeRequest CacheObject `json:"beforeRequest,omitempty"` + // optional State of a cache entry after the request. Leave out this field if + // the information is not available. + AfterRequest CacheObject `json:"afterRequest,omitempty"` + // optional (new in 1.2) A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// CacheObject is used by both beforeRequest and afterRequest +type CacheObject struct { + // optional - Expiration time of the cache entry. + Expires string `json:"expires,omitempty"` + // The last time the cache entry was opened. + LastAccess string `json:"lastAccess"` + // Etag + ETag string `json:"eTag"` + // The number of times the cache entry has been opened. + HitCount int `json:"hitCount"` + // optional (new in 1.2) A comment provided by the user or the application. + Comment string `json:"comment,omitempty"` +} + +// PageTimings describes various phases within request-response round trip. +// All times are specified in milliseconds. +type PageTimings struct { + Blocked int `json:"blocked,omitempty"` + // optional - Time spent in a queue waiting for a network connection. Use -1 + // if the timing does not apply to the current request. + DNS int `json:"dns,omitempty"` + // optional - DNS resolution time. The time required to resolve a host name. + // Use -1 if the timing does not apply to the current request. + Connect int `json:"connect,omitempty"` + // optional - Time required to create TCP connection. Use -1 if the timing + // does not apply to the current request. + Send int `json:"send"` + // Time required to send HTTP request to the server. + Wait int `json:"wait"` + // Waiting for a response from the server. + Receive int `json:"receive"` + // Time required to read entire response from the server (or cache). + Ssl int `json:"ssl,omitempty"` + // optional (new in 1.2) - Time required for SSL/TLS negotiation. If this + // field is defined then the time is also included in the connect field (to + // ensure backward compatibility with HAR 1.1). Use -1 if the timing does not + // apply to the current request. + Comment string `json:"comment,omitempty"` + // optional (new in 1.2) - A comment provided by the user or the application. +} + +// TestResult contains results for an individual HTTP request +type TestResult struct { + URL string `json:"url"` + Status int `json:"status"` // 200, 500, etc. + StartTime time.Time `json:"startTime"` + EndTime time.Time `json:"endTime"` + Latency int `json:"latency"` // milliseconds + Method string `json:"method"` + HarFile string `json:"harfile"` +} + +// aliases for martian lib compatibility + +type Header = NVP +type QueryString = NVP +type Param = PostParam +type Timings = PageTimings diff --git a/agent/pkg/har/types_test.go b/agent/pkg/har/types_test.go new file mode 100644 index 000000000..20e67a9b7 --- /dev/null +++ b/agent/pkg/har/types_test.go @@ -0,0 +1,38 @@ +package har + +import "testing" + +func TestContentEncoded(t *testing.T) { + testCases := []struct { + text string + isBinary bool + expectedStr string + binaryLen int + }{ + {"not-base64", false, "not-base64", 10}, + {"dGVzdA==", false, "test", 4}, + {"test", true, "\f@A", 3}, // valid UTF-8 with some non-printable chars + {"IsDggPCAgPiAgID8gICAgN/vv/e/v/u/v7/9v7+/vyIKIu+3kO+3ke+3ku+3k++3lO+3le+3lu+3l++3mO+3me+3mu+3m++3nO+3ne+3nu+3n++3oO+3oe+3ou+3o++3pO+3pe+3pu+3p++3qO+3qe+3qu+3q++3rO+3re+3ru+3ryIK", true, "test", 132}, // invalid UTF-8 (thus binary), taken from https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt + } + + for _, tc := range testCases { + c := Content{ + Encoding: "base64", + Text: tc.text, + } + isBinary, asBytes, asString := c.B64Decoded() + _ = asBytes + + if tc.isBinary != isBinary { + t.Errorf("Binary flag mismatch: %t != %t", tc.isBinary, isBinary) + } + + if !isBinary && tc.expectedStr != asString { + t.Errorf("Decode value mismatch: %s != %s", tc.expectedStr, asString) + } + + if tc.binaryLen != len(asBytes) { + t.Errorf("Binary len mismatch: %d != %d", tc.binaryLen, len(asBytes)) + } + } +} diff --git a/agent/pkg/utils/har.go b/agent/pkg/har/utils.go similarity index 77% rename from agent/pkg/utils/har.go rename to agent/pkg/har/utils.go index 108e91b7b..a64b107f9 100644 --- a/agent/pkg/utils/har.go +++ b/agent/pkg/har/utils.go @@ -1,4 +1,4 @@ -package utils +package har import ( "bytes" @@ -8,7 +8,6 @@ import ( "strings" "time" - "github.com/google/martian/har" "github.com/up9inc/mizu/shared/logger" ) @@ -55,14 +54,14 @@ import ( // return cookies //} -func BuildHeaders(rawHeaders []interface{}) ([]har.Header, string, string, string, string, string) { +func BuildHeaders(rawHeaders []interface{}) ([]Header, string, string, string, string, string) { var host, scheme, authority, path, status string - headers := make([]har.Header, 0, len(rawHeaders)) + headers := make([]Header, 0, len(rawHeaders)) for _, header := range rawHeaders { h := header.(map[string]interface{}) - headers = append(headers, har.Header{ + headers = append(headers, Header{ Name: h["name"].(string), Value: h["value"].(string), }) @@ -87,8 +86,8 @@ func BuildHeaders(rawHeaders []interface{}) ([]har.Header, string, string, strin return headers, host, scheme, authority, path, status } -func BuildPostParams(rawParams []interface{}) []har.Param { - params := make([]har.Param, 0, len(rawParams)) +func BuildPostParams(rawParams []interface{}) []Param { + params := make([]Param, 0, len(rawParams)) for _, param := range rawParams { p := param.(map[string]interface{}) name := "" @@ -108,10 +107,10 @@ func BuildPostParams(rawParams []interface{}) []har.Param { contentType = p["contentType"].(string) } - params = append(params, har.Param{ + params = append(params, Param{ Name: name, Value: value, - Filename: fileName, + FileName: fileName, ContentType: contentType, }) } @@ -119,9 +118,9 @@ func BuildPostParams(rawParams []interface{}) []har.Param { return params } -func NewRequest(request map[string]interface{}) (harRequest *har.Request, err error) { +func NewRequest(request map[string]interface{}) (harRequest *Request, err error) { headers, host, scheme, authority, path, _ := BuildHeaders(request["_headers"].([]interface{})) - cookies := make([]har.Cookie, 0) // BuildCookies(request["_cookies"].([]interface{})) + cookies := make([]Cookie, 0) // BuildCookies(request["_cookies"].([]interface{})) postData, _ := request["postData"].(map[string]interface{}) mimeType, _ := postData["mimeType"] @@ -134,10 +133,10 @@ func NewRequest(request map[string]interface{}) (harRequest *har.Request, err er postDataText = text.(string) } - queryString := make([]har.QueryString, 0) + queryString := make([]QueryString, 0) for _, _qs := range request["_queryString"].([]interface{}) { qs := _qs.(map[string]interface{}) - queryString = append(queryString, har.QueryString{ + queryString = append(queryString, QueryString{ Name: qs["name"].(string), Value: qs["value"].(string), }) @@ -148,21 +147,21 @@ func NewRequest(request map[string]interface{}) (harRequest *har.Request, err er url = fmt.Sprintf("%s://%s%s", scheme, authority, path) } - harParams := make([]har.Param, 0) + harParams := make([]Param, 0) if postData["params"] != nil { harParams = BuildPostParams(postData["params"].([]interface{})) } - harRequest = &har.Request{ + harRequest = &Request{ Method: request["method"].(string), URL: url, HTTPVersion: request["httpVersion"].(string), - HeadersSize: -1, - BodySize: int64(bytes.NewBufferString(postDataText).Len()), + HeaderSize: -1, + BodySize: bytes.NewBufferString(postDataText).Len(), QueryString: queryString, Headers: headers, Cookies: cookies, - PostData: &har.PostData{ + PostData: PostData{ MimeType: mimeType.(string), Params: harParams, Text: postDataText, @@ -172,9 +171,9 @@ func NewRequest(request map[string]interface{}) (harRequest *har.Request, err er return } -func NewResponse(response map[string]interface{}) (harResponse *har.Response, err error) { +func NewResponse(response map[string]interface{}) (harResponse *Response, err error) { headers, _, _, _, _, _status := BuildHeaders(response["_headers"].([]interface{})) - cookies := make([]har.Cookie, 0) // BuildCookies(response["_cookies"].([]interface{})) + cookies := make([]Cookie, 0) // BuildCookies(response["_cookies"].([]interface{})) content, _ := response["content"].(map[string]interface{}) mimeType, _ := content["mimeType"] @@ -188,11 +187,11 @@ func NewResponse(response map[string]interface{}) (harResponse *har.Response, er bodyText = text.(string) } - harContent := &har.Content{ + harContent := &Content{ Encoding: encoding.(string), MimeType: mimeType.(string), - Text: []byte(bodyText), - Size: int64(len(bodyText)), + Text: bodyText, + Size: len(bodyText), } status := int(response["status"].(float64)) @@ -206,20 +205,20 @@ func NewResponse(response map[string]interface{}) (harResponse *har.Response, er } } - harResponse = &har.Response{ + harResponse = &Response{ HTTPVersion: response["httpVersion"].(string), Status: status, StatusText: response["statusText"].(string), HeadersSize: -1, - BodySize: int64(bytes.NewBufferString(bodyText).Len()), + BodySize: bytes.NewBufferString(bodyText).Len(), Headers: headers, Cookies: cookies, - Content: harContent, + Content: *harContent, } return } -func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*har.Entry, error) { +func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*Entry, error) { harRequest, err := NewRequest(request) if err != nil { logger.Log.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err) @@ -236,16 +235,16 @@ func NewEntry(request map[string]interface{}, response map[string]interface{}, s elapsedTime = 1 } - harEntry := har.Entry{ - StartedDateTime: startTime, - Time: elapsedTime, - Request: harRequest, - Response: harResponse, - Cache: &har.Cache{}, - Timings: &har.Timings{ + harEntry := Entry{ + StartedDateTime: startTime.Format(time.RFC3339), + Time: int(elapsedTime), + Request: *harRequest, + Response: *harResponse, + Cache: Cache{}, + PageTimings: PageTimings{ Send: -1, Wait: -1, - Receive: elapsedTime, + Receive: int(elapsedTime), }, } diff --git a/agent/pkg/models/models.go b/agent/pkg/models/models.go index 7f9ee101b..ba4ca0290 100644 --- a/agent/pkg/models/models.go +++ b/agent/pkg/models/models.go @@ -2,11 +2,10 @@ package models import ( "encoding/json" + tapApi "github.com/up9inc/mizu/tap/api" + "mizuserver/pkg/har" "mizuserver/pkg/rules" - tapApi "github.com/up9inc/mizu/tap/api" - - "github.com/google/martian/har" basenine "github.com/up9inc/basenine/client/go" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/tap" diff --git a/agent/pkg/oas/feeder_test.go b/agent/pkg/oas/feeder_test.go index fa4828708..e3be929e8 100644 --- a/agent/pkg/oas/feeder_test.go +++ b/agent/pkg/oas/feeder_test.go @@ -4,10 +4,10 @@ import ( "bufio" "encoding/json" "errors" - "github.com/google/martian/har" "github.com/up9inc/mizu/shared/logger" "io" "io/ioutil" + "mizuserver/pkg/har" "os" "path/filepath" "sort" @@ -96,7 +96,7 @@ func feedFromHAR(file string) error { } for _, entry := range harDoc.Log.Entries { - GetOasGeneratorInstance().PushEntry(entry) + GetOasGeneratorInstance().PushEntry(&entry) } return nil diff --git a/agent/pkg/oas/oas_generator.go b/agent/pkg/oas/oas_generator.go index 7fc7858f8..bc7e305d3 100644 --- a/agent/pkg/oas/oas_generator.go +++ b/agent/pkg/oas/oas_generator.go @@ -3,8 +3,8 @@ package oas import ( "context" "encoding/json" - "github.com/google/martian/har" "github.com/up9inc/mizu/shared/logger" + "mizuserver/pkg/har" "net/url" "sync" ) @@ -17,7 +17,7 @@ var ( func GetOasGeneratorInstance() *oasGenerator { syncOnce.Do(func() { instance = newOasGenerator() - logger.Log.Debug("Oas Generator Initialized") + logger.Log.Debug("OAS Generator Initialized") }) return instance } diff --git a/agent/pkg/oas/specgen.go b/agent/pkg/oas/specgen.go index b412d4555..3989fd59f 100644 --- a/agent/pkg/oas/specgen.go +++ b/agent/pkg/oas/specgen.go @@ -1,14 +1,13 @@ package oas import ( - "encoding/base64" "encoding/json" "errors" "github.com/chanced/openapi" - "github.com/google/martian/har" "github.com/google/uuid" "github.com/up9inc/mizu/shared/logger" "mime" + "mizuserver/pkg/har" "net/url" "strconv" "strings" @@ -178,7 +177,7 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) { logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path) } - ctype := getRespCtype(entry.Response) + ctype := getRespCtype(&entry.Response) if isCtypeIgnored(ctype) { logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype) } @@ -216,12 +215,12 @@ func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation return nil, nil } - err = handleRequest(entry.Request, opObj, isSuccess) + err = handleRequest(&entry.Request, opObj, isSuccess) if err != nil { return nil, err } - err = handleResponse(entry.Response, opObj, isSuccess) + err = handleResponse(&entry.Response, opObj, isSuccess) if err != nil { return nil, err } @@ -252,7 +251,7 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e } handleNameVals(hdrGW, &opObj.Parameters) - if req.PostData != nil && req.PostData.Text != "" && isSuccess { + if req.PostData.Text != "" && isSuccess { reqBody, err := getRequestBody(req, opObj, isSuccess) if err != nil { return err @@ -342,43 +341,34 @@ func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err } var text string + var isBinary bool if reqResp.Req != nil { - text = reqResp.Req.PostData.Text + isBinary, _, text = reqResp.Req.PostData.B64Decoded() } else { - text = decRespText(reqResp.Resp.Content) + isBinary, _, text = reqResp.Resp.Content.B64Decoded() } - var exampleMsg []byte - // try treating it as json - any, isJSON := anyJSON(text) - if isJSON { - // re-marshal with forced indent - exampleMsg, err = json.MarshalIndent(any, "", "\t") - if err != nil { - panic("Failed to re-marshal value, super-strange") - } - } else { - exampleMsg, err = json.Marshal(text) - if err != nil { - return nil, err - } - } - - content.Example = exampleMsg - return respContent[ctype], nil -} - -func decRespText(content *har.Content) (res string) { - res = string(content.Text) - if content.Encoding == "base64" { - data, err := base64.StdEncoding.DecodeString(res) - if err != nil { - logger.Log.Warningf("error decoding response text as base64: %s", err) + if !isBinary { + var exampleMsg []byte + // try treating it as json + any, isJSON := anyJSON(text) + if isJSON { + // re-marshal with forced indent + exampleMsg, err = json.MarshalIndent(any, "", "\t") + if err != nil { + panic("Failed to re-marshal value, super-strange") + } } else { - res = string(data) + exampleMsg, err = json.Marshal(text) + if err != nil { + return nil, err + } } + + content.Example = exampleMsg } - return + + return respContent[ctype], nil } func getRespCtype(resp *har.Response) string { diff --git a/agent/pkg/oas/specgen_test.go b/agent/pkg/oas/specgen_test.go index 282c6ff4b..9a55a6332 100644 --- a/agent/pkg/oas/specgen_test.go +++ b/agent/pkg/oas/specgen_test.go @@ -3,9 +3,9 @@ package oas import ( "encoding/json" "github.com/chanced/openapi" - "github.com/google/martian/har" "github.com/up9inc/mizu/shared/logger" "io/ioutil" + "mizuserver/pkg/har" "os" "strings" "testing" diff --git a/agent/pkg/oas/tree.go b/agent/pkg/oas/tree.go index 868f7126c..7aaa87d5e 100644 --- a/agent/pkg/oas/tree.go +++ b/agent/pkg/oas/tree.go @@ -27,7 +27,7 @@ func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Nod chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk) var paramObj *openapi.ParameterObj - if chunkIsParam && pathObjToSet != nil { + if chunkIsParam && pathObjToSet != nil && pathObjToSet.Parameters != nil { paramObj = findParamByName(pathObjToSet.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1]) } @@ -82,13 +82,15 @@ func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Nod func (n *Node) createParam() *openapi.ParameterObj { name := "param" - // REST assumption, not always correct - if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 { - name = *n.constant - name = name[:len(name)-2] + "Id" - } else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 { - name = *n.constant - name = name[:len(name)-1] + "Id" + if n.constant != nil { // the node is already a param + // REST assumption, not always correct + if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 { + name = *n.constant + name = name[:len(name)-2] + "Id" + } else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 { + name = *n.constant + name = name[:len(name)-1] + "Id" + } } newParam := createSimpleParam(name, "path", "string") diff --git a/agent/pkg/oas/tree_test.go b/agent/pkg/oas/tree_test.go new file mode 100644 index 000000000..e6fe33066 --- /dev/null +++ b/agent/pkg/oas/tree_test.go @@ -0,0 +1,26 @@ +package oas + +import ( + "github.com/chanced/openapi" + "strings" + "testing" +) + +func TestTree(t *testing.T) { + testCases := []struct { + inp string + }{ + {"/"}, + {"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation"}, + } + + tree := new(Node) + for _, tc := range testCases { + split := strings.Split(tc.inp, "/") + node := tree.getOrSet(split, new(openapi.PathObj)) + + if node.constant == nil { + t.Errorf("nil constant: %s", tc.inp) + } + } +} diff --git a/agent/pkg/oas/utils.go b/agent/pkg/oas/utils.go index ff938ec93..86fb481ed 100644 --- a/agent/pkg/oas/utils.go +++ b/agent/pkg/oas/utils.go @@ -4,8 +4,8 @@ import ( "encoding/json" "errors" "github.com/chanced/openapi" - "github.com/google/martian/har" "github.com/up9inc/mizu/shared/logger" + "mizuserver/pkg/har" "strconv" "strings" ) diff --git a/agent/pkg/rules/rulesHTTP.go b/agent/pkg/rules/rulesHTTP.go index 61550fbcf..eeb71036b 100644 --- a/agent/pkg/rules/rulesHTTP.go +++ b/agent/pkg/rules/rulesHTTP.go @@ -4,15 +4,15 @@ import ( "encoding/base64" "encoding/json" "fmt" + "mizuserver/pkg/har" "reflect" "regexp" "strings" "github.com/up9inc/mizu/shared/logger" - "github.com/google/martian/har" "github.com/up9inc/mizu/shared" - jsonpath "github.com/yalp/jsonpath" + "github.com/yalp/jsonpath" ) type RulesMatched struct { diff --git a/agent/pkg/up9/main.go b/agent/pkg/up9/main.go index 54405b5e1..0a61619de 100644 --- a/agent/pkg/up9/main.go +++ b/agent/pkg/up9/main.go @@ -3,10 +3,10 @@ package up9 import ( "bytes" "compress/zlib" - "encoding/base64" "encoding/json" "fmt" "io/ioutil" + "mizuserver/pkg/har" "mizuserver/pkg/utils" "net/http" "net/url" @@ -15,7 +15,6 @@ import ( "sync" "time" - "github.com/google/martian/har" basenine "github.com/up9inc/basenine/client/go" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" @@ -247,7 +246,7 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva if err := json.Unmarshal([]byte(dataBytes), &entry); err != nil { continue } - harEntry, err := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime) + harEntry, err := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime) if err != nil { continue } @@ -259,11 +258,6 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entry.Destination.Name) } - // go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64 - if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil { - continue - } - batch = append(batch, *harEntry) now := time.Now() diff --git a/tap/api/api.go b/tap/api/api.go index eda989caf..0c5054adc 100644 --- a/tap/api/api.go +++ b/tap/api/api.go @@ -6,12 +6,11 @@ import ( "encoding/json" "errors" "fmt" + "github.com/google/martian/har" "io/ioutil" "net/http" "sync" "time" - - "github.com/google/martian/har" ) type Protocol struct {