Have own HAR library code (#695)

agent/pkg/api/main.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"mizuserver/pkg/har"
 	"mizuserver/pkg/holder"
 	"mizuserver/pkg/providers"
 	"os"
@@ -15,11 +16,9 @@ import (
 
 	"mizuserver/pkg/servicemap"
 
-	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
-
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/oas"
 	"mizuserver/pkg/resolver"
@@ -132,7 +131,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 				mizuEntry.ContractContent = contract.Content
 			}
 
-			harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
+			harEntry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
 			if err == nil {
 				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name)
 				mizuEntry.Rules = rules

agent/pkg/controllers/entries_controller.go

@@ -2,8 +2,8 @@ package controllers
 
 import (
 	"encoding/json"
+	"mizuserver/pkg/har"
 	"mizuserver/pkg/models"
-	"mizuserver/pkg/utils"
 	"mizuserver/pkg/validation"
 	"net/http"
 	"strconv"
@@ -127,7 +127,7 @@ func GetEntry(c *gin.Context) {
 	var rules []map[string]interface{}
 	var isRulesEnabled bool
 	if entry.Protocol.Name == "http" {
-		harEntry, _ := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
+		harEntry, _ := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
 		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
 		isRulesEnabled = _isRulesEnabled
 		inrec, _ := json.Marshal(rulesMatched)

agent/pkg/har/types.go

@@ -0,0 +1,375 @@
+package har
+
+import (
+	"encoding/base64"
+	"github.com/up9inc/mizu/shared/logger"
+	"time"
+	"unicode/utf8"
+)
+
+/*
+HTTP Archive (HAR) format
+https://w3c.github.io/web-performance/specs/HAR/Overview.html
+*/
+
+// HAR is a container type for deserialization
+type HAR struct {
+	Log Log `json:"log"`
+}
+
+// Log represents the root of the exported data. This object MUST be present and its name MUST be "log".
+type Log struct {
+	// The object contains the following name/value pairs:
+
+	// Required. Version number of the format.
+	Version string `json:"version"`
+	// Required. An object of type creator that contains the name and version
+	// information of the log creator application.
+	Creator Creator `json:"creator"`
+	// Optional. An object of type browser that contains the name and version
+	// information of the user agent.
+	Browser Browser `json:"browser"`
+	// Optional. An array of objects of type page, each representing one exported
+	// (tracked) page. Leave out this field if the application does not support
+	// grouping by pages.
+	Pages []Page `json:"pages,omitempty"`
+	// Required. An array of objects of type entry, each representing one
+	// exported (tracked) HTTP request.
+	Entries []Entry `json:"entries"`
+	// Optional. A comment provided by the user or the application. Sorting
+	// entries by startedDateTime (starting from the oldest) is preferred way how
+	// to export data since it can make importing faster. However the reader
+	// application should always make sure the array is sorted (if required for
+	// the import).
+	Comment string `json:"comment"`
+}
+
+// Creator contains information about the log creator application
+type Creator struct {
+	// Required. The name of the application that created the log.
+	Name string `json:"name"`
+	// Required. The version number of the application that created the log.
+	Version string `json:"version"`
+	// Optional. A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Browser that created the log
+type Browser struct {
+	// Required. The name of the browser that created the log.
+	Name string `json:"name"`
+	// Required. The version number of the browser that created the log.
+	Version string `json:"version"`
+	// Optional. A comment provided by the user or the browser.
+	Comment string `json:"comment"`
+}
+
+// Page object for every exported web page and one <entry> object for every HTTP request.
+// In case when an HTTP trace tool isn't able to group requests by a page,
+// the <pages> object is empty and individual requests doesn't have a parent page.
+type Page struct {
+	/* There is one <page> object for every exported web page and one <entry>
+	object for every HTTP request. In case when an HTTP trace tool isn't able to
+	group requests by a page, the <pages> object is empty and individual
+	requests doesn't have a parent page.
+	*/
+
+	// Date and time stamp for the beginning of the page load
+	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00).
+	StartedDateTime string `json:"startedDateTime"`
+	// Unique identifier of a page within the . Entries use it to refer the parent page.
+	ID string `json:"id"`
+	// Page title.
+	Title string `json:"title"`
+	// Detailed timing info about page load.
+	PageTiming PageTiming `json:"pageTiming"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// PageTiming describes timings for various events (states) fired during the page load.
+// All times are specified in milliseconds. If a time info is not available appropriate field is set to -1.
+type PageTiming struct {
+	// Content of the page loaded. Number of milliseconds since page load started
+	// (page.startedDateTime). Use -1 if the timing does not apply to the current
+	// request.
+	// Depeding on the browser, onContentLoad property represents DOMContentLoad
+	// event or document.readyState == interactive.
+	OnContentLoad int `json:"onContentLoad"`
+	// Page is loaded (onLoad event fired). Number of milliseconds since page
+	// load started (page.startedDateTime). Use -1 if the timing does not apply
+	// to the current request.
+	OnLoad int `json:"onLoad"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment"`
+}
+
+// Entry is a unique, optional Reference to the parent page.
+// Leave out this field if the application does not support grouping by pages.
+type Entry struct {
+	Pageref string `json:"pageref,omitempty"`
+	// Date and time stamp of the request start
+	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD).
+	StartedDateTime string `json:"startedDateTime"`
+	// Total elapsed time of the request in milliseconds. This is the sum of all
+	// timings available in the timings object (i.e. not including -1 values) .
+	Time int `json:"time"`
+	// Detailed info about the request.
+	Request Request `json:"request"`
+	// Detailed info about the response.
+	Response Response `json:"response"`
+	// Info about cache usage.
+	Cache Cache `json:"cache"`
+	// Detailed timing info about request/response round trip.
+	PageTimings PageTimings `json:"pageTimings"`
+	// optional (new in 1.2) IP address of the server that was connected
+	// (result of DNS resolution).
+	ServerIPAddress string `json:"serverIPAddress,omitempty"`
+	// optional (new in 1.2) Unique ID of the parent TCP/IP connection, can be
+	// the client port number. Note that a port number doesn't have to be unique
+	// identifier in cases where the port is shared for more connections. If the
+	// port isn't available for the application, any other unique connection ID
+	// can be used instead (e.g. connection index). Leave out this field if the
+	// application doesn't support this info.
+	Connection string `json:"connection,omitempty"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Request contains detailed info about performed request.
+type Request struct {
+	// Request method (GET, POST, ...).
+	Method string `json:"method"`
+	// Absolute URL of the request (fragments are not included).
+	URL string `json:"url"`
+	// Request HTTP Version.
+	HTTPVersion string `json:"httpVersion"`
+	// List of cookie objects.
+	Cookies []Cookie `json:"cookies"`
+	// List of header objects.
+	Headers []NVP `json:"headers"`
+	// List of query parameter objects.
+	QueryString []NVP `json:"queryString"`
+	// Posted data.
+	PostData PostData `json:"postData"`
+	// Total number of bytes from the start of the HTTP request message until
+	// (and including) the double CRLF before the body. Set to -1 if the info
+	// is not available.
+	HeaderSize int `json:"headerSize"`
+	// Size of the request body (POST data payload) in bytes. Set to -1 if the
+	// info is not available.
+	BodySize int `json:"bodySize"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment"`
+}
+
+// Response contains detailed info about the response.
+type Response struct {
+	// Response status.
+	Status int `json:"status"`
+	// Response status description.
+	StatusText string `json:"statusText"`
+	// Response HTTP Version.
+	HTTPVersion string `json:"httpVersion"`
+	// List of cookie objects.
+	Cookies []Cookie `json:"cookies"`
+	// List of header objects.
+	Headers []NVP `json:"headers"`
+	// Details about the response body.
+	Content Content `json:"content"`
+	// Redirection target URL from the Location response header.
+	RedirectURL string `json:"redirectURL"`
+	// Total number of bytes from the start of the HTTP response message until
+	// (and including) the double CRLF before the body. Set to -1 if the info is
+	// not available.
+	// The size of received response-headers is computed only from headers that
+	// are really received from the server. Additional headers appended by the
+	// browser are not included in this number, but they appear in the list of
+	// header objects.
+	HeadersSize int `json:"headersSize"`
+	// Size of the received response body in bytes. Set to zero in case of
+	// responses coming from the cache (304). Set to -1 if the info is not
+	// available.
+	BodySize int `json:"bodySize"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Cookie contains list of all cookies (used in <request> and <response> objects).
+type Cookie struct {
+	// The name of the cookie.
+	Name string `json:"name"`
+	// The cookie value.
+	Value string `json:"value"`
+	// optional The path pertaining to the cookie.
+	Path string `json:"path,omitempty"`
+	// optional The host of the cookie.
+	Domain string `json:"domain,omitempty"`
+	// optional Cookie expiration time.
+	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.123+02:00).
+	Expires string `json:"expires,omitempty"`
+	// optional Set to true if the cookie is HTTP only, false otherwise.
+	HTTPOnly bool `json:"httpOnly,omitempty"`
+	// optional (new in 1.2) True if the cookie was transmitted over ssl, false
+	// otherwise.
+	Secure bool `json:"secure,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// NVP is simply a name/value pair with a comment
+type NVP struct {
+	Name    string `json:"name"`
+	Value   string `json:"value"`
+	Comment string `json:"comment,omitempty"`
+}
+
+// PostData describes posted data, if any (embedded in <request> object).
+type PostData struct {
+	//  Mime type of posted data.
+	MimeType string `json:"mimeType"`
+	//  List of posted parameters (in case of URL encoded parameters).
+	Params []PostParam `json:"params"`
+	//  Plain text posted data
+	Text string `json:"text"`
+	// optional (new in 1.2) A comment provided by the user or the
+	// application.
+	Comment string `json:"comment,omitempty"`
+}
+
+func (d PostData) B64Decoded() (bool, []byte, string) {
+	// there is a weird gap in HAR spec 1.2, that does not define encoding for binary POST bodies
+	// we have own convention of putting `base64` into comment field to handle it similar to response `Content`
+	return b64Decoded(d.Comment, d.Text)
+}
+
+// PostParam is a list of posted parameters, if any (embedded in <postData> object).
+type PostParam struct {
+	// name of a posted parameter.
+	Name string `json:"name"`
+	// optional value of a posted parameter or content of a posted file.
+	Value string `json:"value,omitempty"`
+	// optional name of a posted file.
+	FileName string `json:"fileName,omitempty"`
+	// optional content type of a posted file.
+	ContentType string `json:"contentType,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Content describes details about response content (embedded in <response> object).
+type Content struct {
+	// Length of the returned content in bytes. Should be equal to
+	// response.bodySize if there is no compression and bigger when the content
+	// has been compressed.
+	Size int `json:"size"`
+	// optional Number of bytes saved. Leave out this field if the information
+	// is not available.
+	Compression int `json:"compression,omitempty"`
+	// MIME type of the response text (value of the Content-Type response
+	// header). The charset attribute of the MIME type is included (if
+	// available).
+	MimeType string `json:"mimeType"`
+	// optional Response body sent from the server or loaded from the browser
+	// cache. This field is populated with textual content only. The text field
+	// is either HTTP decoded text or a encoded (e.g. "base64") representation of
+	// the response body. Leave out this field if the information is not
+	// available.
+	Text string `json:"text,omitempty"`
+	// optional (new in 1.2) Encoding used for response text field e.g
+	// "base64". Leave out this field if the text field is HTTP decoded
+	// (decompressed & unchunked), than trans-coded from its original character
+	// set into UTF-8.
+	Encoding string `json:"encoding,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+func (c Content) B64Decoded() (bool, []byte, string) {
+	return b64Decoded(c.Encoding, c.Text)
+}
+
+func b64Decoded(enc string, text string) (isBinary bool, asBytes []byte, asString string) {
+	if enc == "base64" {
+		decoded, err := base64.StdEncoding.DecodeString(text)
+		if err != nil {
+			logger.Log.Warningf("Failed to decode content as base64: %s", text)
+			return false, []byte(text), text
+		}
+		valid := utf8.Valid(decoded)
+		return !valid, decoded, string(decoded)
+	} else {
+		return false, nil, text
+	}
+}
+
+// Cache contains info about a request coming from browser cache.
+type Cache struct {
+	// optional State of a cache entry before the request. Leave out this field
+	// if the information is not available.
+	BeforeRequest CacheObject `json:"beforeRequest,omitempty"`
+	// optional State of a cache entry after the request. Leave out this field if
+	// the information is not available.
+	AfterRequest CacheObject `json:"afterRequest,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// CacheObject is used by both beforeRequest and afterRequest
+type CacheObject struct {
+	// optional - Expiration time of the cache entry.
+	Expires string `json:"expires,omitempty"`
+	// The last time the cache entry was opened.
+	LastAccess string `json:"lastAccess"`
+	// Etag
+	ETag string `json:"eTag"`
+	// The number of times the cache entry has been opened.
+	HitCount int `json:"hitCount"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// PageTimings describes various phases within request-response round trip.
+// All times are specified in milliseconds.
+type PageTimings struct {
+	Blocked int `json:"blocked,omitempty"`
+	// optional - Time spent in a queue waiting for a network connection. Use -1
+	// if the timing does not apply to the current request.
+	DNS int `json:"dns,omitempty"`
+	// optional - DNS resolution time. The time required to resolve a host name.
+	// Use -1 if the timing does not apply to the current request.
+	Connect int `json:"connect,omitempty"`
+	// optional - Time required to create TCP connection. Use -1 if the timing
+	// does not apply to the current request.
+	Send int `json:"send"`
+	// Time required to send HTTP request to the server.
+	Wait int `json:"wait"`
+	// Waiting for a response from the server.
+	Receive int `json:"receive"`
+	// Time required to read entire response from the server (or cache).
+	Ssl int `json:"ssl,omitempty"`
+	// optional (new in 1.2) - Time required for SSL/TLS negotiation. If this
+	// field is defined then the time is also included in the connect field (to
+	// ensure backward compatibility with HAR 1.1). Use -1 if the timing does not
+	// apply to the current request.
+	Comment string `json:"comment,omitempty"`
+	// optional (new in 1.2) - A comment provided by the user or the application.
+}
+
+// TestResult contains results for an individual HTTP request
+type TestResult struct {
+	URL       string    `json:"url"`
+	Status    int       `json:"status"` // 200, 500, etc.
+	StartTime time.Time `json:"startTime"`
+	EndTime   time.Time `json:"endTime"`
+	Latency   int       `json:"latency"` // milliseconds
+	Method    string    `json:"method"`
+	HarFile   string    `json:"harfile"`
+}
+
+// aliases for martian lib compatibility
+
+type Header = NVP
+type QueryString = NVP
+type Param = PostParam
+type Timings = PageTimings

agent/pkg/har/types_test.go

@@ -0,0 +1,38 @@
+package har
+
+import "testing"
+
+func TestContentEncoded(t *testing.T) {
+	testCases := []struct {
+		text        string
+		isBinary    bool
+		expectedStr string
+		binaryLen   int
+	}{
+		{"not-base64", false, "not-base64", 10},
+		{"dGVzdA==", false, "test", 4},
+		{"test", true, "\f@A", 3}, // valid UTF-8 with some non-printable chars
+		{"IsDggPCAgPiAgID8gICAgN/vv/e/v/u/v7/9v7+/vyIKIu+3kO+3ke+3ku+3k++3lO+3le+3lu+3l++3mO+3me+3mu+3m++3nO+3ne+3nu+3n++3oO+3oe+3ou+3o++3pO+3pe+3pu+3p++3qO+3qe+3qu+3q++3rO+3re+3ru+3ryIK", true, "test", 132}, // invalid UTF-8 (thus binary), taken from https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
+	}
+
+	for _, tc := range testCases {
+		c := Content{
+			Encoding: "base64",
+			Text:     tc.text,
+		}
+		isBinary, asBytes, asString := c.B64Decoded()
+		_ = asBytes
+
+		if tc.isBinary != isBinary {
+			t.Errorf("Binary flag mismatch: %t != %t", tc.isBinary, isBinary)
+		}
+
+		if !isBinary && tc.expectedStr != asString {
+			t.Errorf("Decode value mismatch: %s != %s", tc.expectedStr, asString)
+		}
+
+		if tc.binaryLen != len(asBytes) {
+			t.Errorf("Binary len mismatch: %d != %d", tc.binaryLen, len(asBytes))
+		}
+	}
+}

agent/pkg/har/utils.go

@@ -1,4 +1,4 @@
-package utils
+package har
 
 import (
 	"bytes"
@@ -8,7 +8,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
 )
 
@@ -55,14 +54,14 @@ import (
 //	return cookies
 //}
 
-func BuildHeaders(rawHeaders []interface{}) ([]har.Header, string, string, string, string, string) {
+func BuildHeaders(rawHeaders []interface{}) ([]Header, string, string, string, string, string) {
 	var host, scheme, authority, path, status string
-	headers := make([]har.Header, 0, len(rawHeaders))
+	headers := make([]Header, 0, len(rawHeaders))
 
 	for _, header := range rawHeaders {
 		h := header.(map[string]interface{})
 
-		headers = append(headers, har.Header{
+		headers = append(headers, Header{
 			Name:  h["name"].(string),
 			Value: h["value"].(string),
 		})
@@ -87,8 +86,8 @@ func BuildHeaders(rawHeaders []interface{}) ([]har.Header, string, string, strin
 	return headers, host, scheme, authority, path, status
 }
 
-func BuildPostParams(rawParams []interface{}) []har.Param {
+func BuildPostParams(rawParams []interface{}) []Param {
-	params := make([]har.Param, 0, len(rawParams))
+	params := make([]Param, 0, len(rawParams))
 	for _, param := range rawParams {
 		p := param.(map[string]interface{})
 		name := ""
@@ -108,10 +107,10 @@ func BuildPostParams(rawParams []interface{}) []har.Param {
 			contentType = p["contentType"].(string)
 		}
 
-		params = append(params, har.Param{
+		params = append(params, Param{
 			Name:        name,
 			Value:       value,
-			Filename:    fileName,
+			FileName:    fileName,
 			ContentType: contentType,
 		})
 	}
@@ -119,9 +118,9 @@ func BuildPostParams(rawParams []interface{}) []har.Param {
 	return params
 }
 
-func NewRequest(request map[string]interface{}) (harRequest *har.Request, err error) {
+func NewRequest(request map[string]interface{}) (harRequest *Request, err error) {
 	headers, host, scheme, authority, path, _ := BuildHeaders(request["_headers"].([]interface{}))
-	cookies := make([]har.Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
+	cookies := make([]Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
 
 	postData, _ := request["postData"].(map[string]interface{})
 	mimeType, _ := postData["mimeType"]
@@ -134,10 +133,10 @@ func NewRequest(request map[string]interface{}) (harRequest *har.Request, err er
 		postDataText = text.(string)
 	}
 
-	queryString := make([]har.QueryString, 0)
+	queryString := make([]QueryString, 0)
 	for _, _qs := range request["_queryString"].([]interface{}) {
 		qs := _qs.(map[string]interface{})
-		queryString = append(queryString, har.QueryString{
+		queryString = append(queryString, QueryString{
 			Name:  qs["name"].(string),
 			Value: qs["value"].(string),
 		})
@@ -148,21 +147,21 @@ func NewRequest(request map[string]interface{}) (harRequest *har.Request, err er
 		url = fmt.Sprintf("%s://%s%s", scheme, authority, path)
 	}
 
-	harParams := make([]har.Param, 0)
+	harParams := make([]Param, 0)
 	if postData["params"] != nil {
 		harParams = BuildPostParams(postData["params"].([]interface{}))
 	}
 
-	harRequest = &har.Request{
+	harRequest = &Request{
 		Method:      request["method"].(string),
 		URL:         url,
 		HTTPVersion: request["httpVersion"].(string),
-		HeadersSize: -1,
+		HeaderSize:  -1,
-		BodySize:    int64(bytes.NewBufferString(postDataText).Len()),
+		BodySize:    bytes.NewBufferString(postDataText).Len(),
 		QueryString: queryString,
 		Headers:     headers,
 		Cookies:     cookies,
-		PostData: &har.PostData{
+		PostData: PostData{
 			MimeType: mimeType.(string),
 			Params:   harParams,
 			Text:     postDataText,
@@ -172,9 +171,9 @@ func NewRequest(request map[string]interface{}) (harRequest *har.Request, err er
 	return
 }
 
-func NewResponse(response map[string]interface{}) (harResponse *har.Response, err error) {
+func NewResponse(response map[string]interface{}) (harResponse *Response, err error) {
 	headers, _, _, _, _, _status := BuildHeaders(response["_headers"].([]interface{}))
-	cookies := make([]har.Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
+	cookies := make([]Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
 
 	content, _ := response["content"].(map[string]interface{})
 	mimeType, _ := content["mimeType"]
@@ -188,11 +187,11 @@ func NewResponse(response map[string]interface{}) (harResponse *har.Response, er
 		bodyText = text.(string)
 	}
 
-	harContent := &har.Content{
+	harContent := &Content{
 		Encoding: encoding.(string),
 		MimeType: mimeType.(string),
-		Text:     []byte(bodyText),
+		Text:     bodyText,
-		Size:     int64(len(bodyText)),
+		Size:     len(bodyText),
 	}
 
 	status := int(response["status"].(float64))
@@ -206,20 +205,20 @@ func NewResponse(response map[string]interface{}) (harResponse *har.Response, er
 		}
 	}
 
-	harResponse = &har.Response{
+	harResponse = &Response{
 		HTTPVersion: response["httpVersion"].(string),
 		Status:      status,
 		StatusText:  response["statusText"].(string),
 		HeadersSize: -1,
-		BodySize:    int64(bytes.NewBufferString(bodyText).Len()),
+		BodySize:    bytes.NewBufferString(bodyText).Len(),
 		Headers:     headers,
 		Cookies:     cookies,
-		Content:     harContent,
+		Content:     *harContent,
 	}
 	return
 }
 
-func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*har.Entry, error) {
+func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*Entry, error) {
 	harRequest, err := NewRequest(request)
 	if err != nil {
 		logger.Log.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err)
@@ -236,16 +235,16 @@ func NewEntry(request map[string]interface{}, response map[string]interface{}, s
 		elapsedTime = 1
 	}
 
-	harEntry := har.Entry{
+	harEntry := Entry{
-		StartedDateTime: startTime,
+		StartedDateTime: startTime.Format(time.RFC3339),
-		Time:            elapsedTime,
+		Time:            int(elapsedTime),
-		Request:         harRequest,
+		Request:         *harRequest,
-		Response:        harResponse,
+		Response:        *harResponse,
-		Cache:           &har.Cache{},
+		Cache:           Cache{},
-		Timings: &har.Timings{
+		PageTimings: PageTimings{
 			Send:    -1,
 			Wait:    -1,
-			Receive: elapsedTime,
+			Receive: int(elapsedTime),
 		},
 	}
 

agent/pkg/models/models.go

@@ -2,11 +2,10 @@ package models
 
 import (
 	"encoding/json"
+	tapApi "github.com/up9inc/mizu/tap/api"
+	"mizuserver/pkg/har"
 	"mizuserver/pkg/rules"
 
-	tapApi "github.com/up9inc/mizu/tap/api"
-
-	"github.com/google/martian/har"
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"

agent/pkg/oas/feeder_test.go

@@ -4,10 +4,10 @@ import (
 	"bufio"
 	"encoding/json"
 	"errors"
-	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
 	"io"
 	"io/ioutil"
+	"mizuserver/pkg/har"
 	"os"
 	"path/filepath"
 	"sort"
@@ -96,7 +96,7 @@ func feedFromHAR(file string) error {
 	}
 
 	for _, entry := range harDoc.Log.Entries {
-		GetOasGeneratorInstance().PushEntry(entry)
+		GetOasGeneratorInstance().PushEntry(&entry)
 	}
 
 	return nil

agent/pkg/oas/oas_generator.go

@@ -3,8 +3,8 @@ package oas
 import (
 	"context"
 	"encoding/json"
-	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/har"
 	"net/url"
 	"sync"
 )
@@ -17,7 +17,7 @@ var (
 func GetOasGeneratorInstance() *oasGenerator {
 	syncOnce.Do(func() {
 		instance = newOasGenerator()
-		logger.Log.Debug("Oas Generator Initialized")
+		logger.Log.Debug("OAS Generator Initialized")
 	})
 	return instance
 }

agent/pkg/oas/specgen.go

@@ -1,14 +1,13 @@
 package oas
 
 import (
-	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"github.com/chanced/openapi"
-	"github.com/google/martian/har"
 	"github.com/google/uuid"
 	"github.com/up9inc/mizu/shared/logger"
 	"mime"
+	"mizuserver/pkg/har"
 	"net/url"
 	"strconv"
 	"strings"
@@ -178,7 +177,7 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
 		logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path)
 	}
 
-	ctype := getRespCtype(entry.Response)
+	ctype := getRespCtype(&entry.Response)
 	if isCtypeIgnored(ctype) {
 		logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype)
 	}
@@ -216,12 +215,12 @@ func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation
 		return nil, nil
 	}
 
-	err = handleRequest(entry.Request, opObj, isSuccess)
+	err = handleRequest(&entry.Request, opObj, isSuccess)
 	if err != nil {
 		return nil, err
 	}
 
-	err = handleResponse(entry.Response, opObj, isSuccess)
+	err = handleResponse(&entry.Response, opObj, isSuccess)
 	if err != nil {
 		return nil, err
 	}
@@ -252,7 +251,7 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e
 	}
 	handleNameVals(hdrGW, &opObj.Parameters)
 
-	if req.PostData != nil && req.PostData.Text != "" && isSuccess {
+	if req.PostData.Text != "" && isSuccess {
 		reqBody, err := getRequestBody(req, opObj, isSuccess)
 		if err != nil {
 			return err
@@ -342,43 +341,34 @@ func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err
 	}
 
 	var text string
+	var isBinary bool
 	if reqResp.Req != nil {
-		text = reqResp.Req.PostData.Text
+		isBinary, _, text = reqResp.Req.PostData.B64Decoded()
 	} else {
-		text = decRespText(reqResp.Resp.Content)
+		isBinary, _, text = reqResp.Resp.Content.B64Decoded()
 	}
 
-	var exampleMsg []byte
+	if !isBinary {
-	// try treating it as json
+		var exampleMsg []byte
-	any, isJSON := anyJSON(text)
+		// try treating it as json
-	if isJSON {
+		any, isJSON := anyJSON(text)
-		// re-marshal with forced indent
+		if isJSON {
-		exampleMsg, err = json.MarshalIndent(any, "", "\t")
+			// re-marshal with forced indent
-		if err != nil {
+			exampleMsg, err = json.MarshalIndent(any, "", "\t")
-			panic("Failed to re-marshal value, super-strange")
+			if err != nil {
-		}
+				panic("Failed to re-marshal value, super-strange")
-	} else {
+			}
-		exampleMsg, err = json.Marshal(text)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	content.Example = exampleMsg
-	return respContent[ctype], nil
-}
-
-func decRespText(content *har.Content) (res string) {
-	res = string(content.Text)
-	if content.Encoding == "base64" {
-		data, err := base64.StdEncoding.DecodeString(res)
-		if err != nil {
-			logger.Log.Warningf("error decoding response text as base64: %s", err)
 		} else {
-			res = string(data)
+			exampleMsg, err = json.Marshal(text)
+			if err != nil {
+				return nil, err
+			}
 		}
+
+		content.Example = exampleMsg
 	}
-	return
+
+	return respContent[ctype], nil
 }
 
 func getRespCtype(resp *har.Response) string {

agent/pkg/oas/specgen_test.go

@@ -3,9 +3,9 @@ package oas
 import (
 	"encoding/json"
 	"github.com/chanced/openapi"
-	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
 	"io/ioutil"
+	"mizuserver/pkg/har"
 	"os"
 	"strings"
 	"testing"

agent/pkg/oas/tree.go

@@ -27,7 +27,7 @@ func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Nod
 	chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk)
 
 	var paramObj *openapi.ParameterObj
-	if chunkIsParam && pathObjToSet != nil {
+	if chunkIsParam && pathObjToSet != nil && pathObjToSet.Parameters != nil {
 		paramObj = findParamByName(pathObjToSet.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
 	}
 
@@ -82,13 +82,15 @@ func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Nod
 func (n *Node) createParam() *openapi.ParameterObj {
 	name := "param"
 
-	// REST assumption, not always correct
+	if n.constant != nil { // the node is already a param
-	if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
+		// REST assumption, not always correct
-		name = *n.constant
+		if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
-		name = name[:len(name)-2] + "Id"
+			name = *n.constant
-	} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
+			name = name[:len(name)-2] + "Id"
-		name = *n.constant
+		} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
-		name = name[:len(name)-1] + "Id"
+			name = *n.constant
+			name = name[:len(name)-1] + "Id"
+		}
 	}
 
 	newParam := createSimpleParam(name, "path", "string")

agent/pkg/oas/tree_test.go

@@ -0,0 +1,26 @@
+package oas
+
+import (
+	"github.com/chanced/openapi"
+	"strings"
+	"testing"
+)
+
+func TestTree(t *testing.T) {
+	testCases := []struct {
+		inp string
+	}{
+		{"/"},
+		{"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation"},
+	}
+
+	tree := new(Node)
+	for _, tc := range testCases {
+		split := strings.Split(tc.inp, "/")
+		node := tree.getOrSet(split, new(openapi.PathObj))
+
+		if node.constant == nil {
+			t.Errorf("nil constant: %s", tc.inp)
+		}
+	}
+}

agent/pkg/oas/utils.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 	"errors"
 	"github.com/chanced/openapi"
-	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/har"
 	"strconv"
 	"strings"
 )

agent/pkg/rules/rulesHTTP.go

@@ -4,15 +4,15 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"mizuserver/pkg/har"
 	"reflect"
 	"regexp"
 	"strings"
 
 	"github.com/up9inc/mizu/shared/logger"
 
-	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared"
-	jsonpath "github.com/yalp/jsonpath"
+	"github.com/yalp/jsonpath"
 )
 
 type RulesMatched struct {

agent/pkg/up9/main.go

@@ -3,10 +3,10 @@ package up9
 import (
 	"bytes"
 	"compress/zlib"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"mizuserver/pkg/har"
 	"mizuserver/pkg/utils"
 	"net/http"
 	"net/url"
@@ -15,7 +15,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/google/martian/har"
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
@@ -247,7 +246,7 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 			if err := json.Unmarshal([]byte(dataBytes), &entry); err != nil {
 				continue
 			}
-			harEntry, err := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
+			harEntry, err := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
 			if err != nil {
 				continue
 			}
@@ -259,11 +258,6 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 				harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entry.Destination.Name)
 			}
 
-			// go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64
-			if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil {
-				continue
-			}
-
 			batch = append(batch, *harEntry)
 
 			now := time.Now()

tap/api/api.go

@@ -6,12 +6,11 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/google/martian/har"
 	"io/ioutil"
 	"net/http"
 	"sync"
 	"time"
-
-	"github.com/google/martian/har"
 )
 
 type Protocol struct {