TRA-3658 - Fix analysis feature (#261)

agent/pkg/controllers/entries_controller.go

@@ -48,7 +48,7 @@ func GetEntries(c *gin.Context) {
 		Find(&entries)
 
 	if len(entries) > 0 && order == database.OrderDesc {
-		// the entries always order from oldest to newest so we should revers
+		// the entries always order from oldest to newest - we should reverse
 		utils.ReverseSlice(entries)
 	}
 
@@ -95,7 +95,7 @@ func GetHARs(c *gin.Context) {
 		Find(&entries)
 
 	if len(entries) > 0 {
-		// the entries always order from oldest to newest so we should revers
+		// the entries always order from oldest to newest - we should reverse
 		utils.ReverseSlice(entries)
 	}
 
@@ -113,7 +113,7 @@ func GetHARs(c *gin.Context) {
 		if sourceOfEntry != "" {
 			// naively assumes the proper service source is http
 			sourceOfEntry = fmt.Sprintf("http://%s", sourceOfEntry)
-			//replace / from the file name cause they end up creating a corrupted folder
+			//replace / from the file name because they end up creating a corrupted folder
 			fileName = fmt.Sprintf("%s.har", strings.ReplaceAll(sourceOfEntry, "/", "_"))
 		} else {
 			fileName = "unknown_source.har"
@@ -202,15 +202,21 @@ func GetFullEntries(c *gin.Context) {
 		timestampTo = entriesFilter.To
 	}
 
-	entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo)
+	entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo, nil)
-	result := make([]models.FullEntryDetails, 0)
+
+	result := make([]har.Entry, 0)
 	for _, data := range entriesArray {
-		harEntry := models.FullEntryDetails{}
+		var pair tapApi.RequestResponsePair
-		if err := models.GetEntry(&data, &harEntry); err != nil {
+		if err := json.Unmarshal([]byte(data.Entry), &pair); err != nil {
 			continue
 		}
-		result = append(result, harEntry)
+		harEntry, err := utils.NewEntry(&pair)
+		if err != nil {
+			continue
+		}
+		result = append(result, *harEntry)
 	}
+
 	c.JSON(http.StatusOK, result)
 }
 
@@ -220,22 +226,6 @@ func GetEntry(c *gin.Context) {
 		Where(map[string]string{"entryId": c.Param("entryId")}).
 		First(&entryData)
 
-	fullEntry := models.FullEntryDetails{}
-	if err := models.GetEntry(&entryData, &fullEntry); err != nil {
-		c.JSON(http.StatusInternalServerError, map[string]interface{}{
-			"error": true,
-			"msg":   "Can't get entry details",
-		})
-	}
-
-	// FIXME: Fix the part below
-	// fullEntryWithPolicy := models.FullEntryWithPolicy{}
-	// if err := models.GetEntry(&entryData, &fullEntryWithPolicy); err != nil {
-	// 	c.JSON(http.StatusInternalServerError, map[string]interface{}{
-	// 		"error": true,
-	// 		"msg":   "Can't get entry details",
-	// 	})
-	// }
 	extension := extensionsMap[entryData.ProtocolName]
 	protocol, representation, bodySize, _ := extension.Dissector.Represent(&entryData)
 	c.JSON(http.StatusOK, tapApi.MizuEntryWrapper{

agent/pkg/database/main.go

@@ -57,10 +57,16 @@ func initDataBase(databasePath string) *gorm.DB {
 	return temp
 }
 
-func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []tapApi.MizuEntry {
+func GetEntriesFromDb(timestampFrom int64, timestampTo int64, protocolName *string) []tapApi.MizuEntry {
 	order := OrderDesc
+	protocolNameCondition := "1 = 1"
+	if protocolName != nil {
+		protocolNameCondition = fmt.Sprintf("protocolKey = '%s'", *protocolName)
+	}
+
 	var entries []tapApi.MizuEntry
 	GetEntriesTable().
+		Where(protocolNameCondition).
 		Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)).
 		Order(fmt.Sprintf("timestamp %s", order)).
 		Find(&entries)

agent/pkg/models/models.go

@@ -2,9 +2,7 @@ package models
 
 import (
 	"encoding/json"
-
 	tapApi "github.com/up9inc/mizu/tap/api"
-
 	"mizuserver/pkg/rules"
 	"mizuserver/pkg/utils"
 
@@ -17,47 +15,14 @@ func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error {
 	return v.UnmarshalData(r)
 }
 
-func NewApplicableRules(status bool, latency int64, number int) tapApi.ApplicableRules {
+// TODO: until we fixed the Rules feature
-	ar := tapApi.ApplicableRules{}
+//func NewApplicableRules(status bool, latency int64, number int) tapApi.ApplicableRules {
-	ar.Status = status
+//	ar := tapApi.ApplicableRules{}
-	ar.Latency = latency
+//	ar.Status = status
-	ar.NumberOfRules = number
+//	ar.Latency = latency
-	return ar
+//	ar.NumberOfRules = number
-}
+//	return ar
-
+//}
-type FullEntryDetails struct {
-	har.Entry
-}
-
-type FullEntryDetailsExtra struct {
-	har.Entry
-}
-
-func (fed *FullEntryDetails) UnmarshalData(entry *tapApi.MizuEntry) error {
-	if err := json.Unmarshal([]byte(entry.Entry), &fed.Entry); err != nil {
-		return err
-	}
-
-	if entry.ResolvedDestination != "" {
-		fed.Entry.Request.URL = utils.SetHostname(fed.Entry.Request.URL, entry.ResolvedDestination)
-	}
-	return nil
-}
-
-func (fedex *FullEntryDetailsExtra) UnmarshalData(entry *tapApi.MizuEntry) error {
-	if err := json.Unmarshal([]byte(entry.Entry), &fedex.Entry); err != nil {
-		return err
-	}
-
-	if entry.ResolvedSource != "" {
-		fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-source", Value: entry.ResolvedSource})
-	}
-	if entry.ResolvedDestination != "" {
-		fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entry.ResolvedDestination})
-		fedex.Entry.Request.URL = utils.SetHostname(fedex.Entry.Request.URL, entry.ResolvedDestination)
-	}
-	return nil
-}
 
 type EntriesFilter struct {
 	Limit     int    `form:"limit" validate:"required,min=1,max=200"`
@@ -147,9 +112,15 @@ type FullEntryWithPolicy struct {
 }
 
 func (fewp *FullEntryWithPolicy) UnmarshalData(entry *tapApi.MizuEntry) error {
-	if err := json.Unmarshal([]byte(entry.Entry), &fewp.Entry); err != nil {
+	var pair tapApi.RequestResponsePair
+	if err := json.Unmarshal([]byte(entry.Entry), &pair); err != nil {
 		return err
 	}
+	harEntry, err := utils.NewEntry(&pair)
+	if err != nil {
+		return err
+	}
+	fewp.Entry = *harEntry
 
 	_, resultPolicyToSend := rules.MatchRequestPolicy(fewp.Entry, entry.Service)
 	fewp.RulesMatched = resultPolicyToSend
@@ -157,9 +128,10 @@ func (fewp *FullEntryWithPolicy) UnmarshalData(entry *tapApi.MizuEntry) error {
 	return nil
 }
 
-func RunValidationRulesState(harEntry har.Entry, service string) tapApi.ApplicableRules {
+// TODO: until we fixed the Rules feature
-	numberOfRules, resultPolicyToSend := rules.MatchRequestPolicy(harEntry, service)
+//func RunValidationRulesState(harEntry har.Entry, service string) tapApi.ApplicableRules {
-	statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend, numberOfRules)
+//	numberOfRules, resultPolicyToSend := rules.MatchRequestPolicy(harEntry, service)
-	ar := NewApplicableRules(statusPolicyToSend, latency, numberOfRules)
+//	statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend, numberOfRules)
-	return ar
+//	ar := NewApplicableRules(statusPolicyToSend, latency, numberOfRules)
-}
+//	return ar
+//}

agent/pkg/up9/main.go

@@ -5,12 +5,14 @@ import (
 	"compress/zlib"
 	"encoding/json"
 	"fmt"
+	"github.com/google/martian/har"
 	"github.com/romana/rlog"
 	"github.com/up9inc/mizu/shared"
+	tapApi "github.com/up9inc/mizu/tap/api"
 	"io/ioutil"
 	"log"
 	"mizuserver/pkg/database"
-	"mizuserver/pkg/models"
+	"mizuserver/pkg/utils"
 	"net/http"
 	"net/url"
 	"strings"
@@ -129,21 +131,33 @@ func UploadEntriesImpl(token string, model string, envPrefix string, sleepInterv
 	for {
 		timestampTo := time.Now().UnixNano() / int64(time.Millisecond)
 		rlog.Infof("Getting entries from %v, to %v\n", timestampFrom, timestampTo)
-		entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo)
+		protocolFilter := "http"
+		entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo, &protocolFilter)
 
 		if len(entriesArray) > 0 {
-
+			result := make([]har.Entry, 0)
-			fullEntriesExtra := make([]models.FullEntryDetailsExtra, 0)
 			for _, data := range entriesArray {
-				harEntry := models.FullEntryDetailsExtra{}
+				var pair tapApi.RequestResponsePair
-				if err := models.GetEntry(&data, &harEntry); err != nil {
+				if err := json.Unmarshal([]byte(data.Entry), &pair); err != nil {
 					continue
 				}
-				fullEntriesExtra = append(fullEntriesExtra, harEntry)
+				harEntry, err := utils.NewEntry(&pair)
+				if err != nil {
+					continue
+				}
+				if data.ResolvedSource != "" {
+					harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: data.ResolvedSource})
+				}
+				if data.ResolvedDestination != "" {
+					harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: data.ResolvedDestination})
+					harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, data.ResolvedDestination)
+				}
+				result = append(result, *harEntry)
 			}
-			rlog.Infof("About to upload %v entries\n", len(fullEntriesExtra))
 
-			body, jMarshalErr := json.Marshal(fullEntriesExtra)
+			rlog.Infof("About to upload %v entries\n", len(result))
+
+			body, jMarshalErr := json.Marshal(result)
 			if jMarshalErr != nil {
 				analyzeInformation.Reset()
 				rlog.Infof("Stopping analyzing")

agent/pkg/utils/har.go

@@ -0,0 +1,259 @@
+package utils
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/tap"
+	"github.com/up9inc/mizu/tap/api"
+)
+
+
+// Keep it because we might want cookies in the future
+//func BuildCookies(rawCookies []interface{}) []har.Cookie {
+//	cookies := make([]har.Cookie, 0, len(rawCookies))
+//
+//	for _, cookie := range rawCookies {
+//		c := cookie.(map[string]interface{})
+//		expiresStr := ""
+//		if c["expires"] != nil {
+//			expiresStr = c["expires"].(string)
+//		}
+//		expires, _ := time.Parse(time.RFC3339, expiresStr)
+//		httpOnly := false
+//		if c["httponly"] != nil {
+//			httpOnly, _ = strconv.ParseBool(c["httponly"].(string))
+//		}
+//		secure := false
+//		if c["secure"] != nil {
+//			secure, _ = strconv.ParseBool(c["secure"].(string))
+//		}
+//		path := ""
+//		if c["path"] != nil {
+//			path = c["path"].(string)
+//		}
+//		domain := ""
+//		if c["domain"] != nil {
+//			domain = c["domain"].(string)
+//		}
+//
+//		cookies = append(cookies, har.Cookie{
+//			Name:        c["name"].(string),
+//			Value:       c["value"].(string),
+//			Path:        path,
+//			Domain:      domain,
+//			HTTPOnly:    httpOnly,
+//			Secure:      secure,
+//			Expires:     expires,
+//			Expires8601: expiresStr,
+//		})
+//	}
+//
+//	return cookies
+//}
+
+func BuildHeaders(rawHeaders []interface{}) ([]har.Header, string, string, string, string, string) {
+	var host, scheme, authority, path, status string
+	headers := make([]har.Header, 0, len(rawHeaders))
+
+	for _, header := range rawHeaders {
+		h := header.(map[string]interface{})
+
+		headers = append(headers, har.Header{
+			Name:  h["name"].(string),
+			Value: h["value"].(string),
+		})
+
+		if h["name"] == "Host" {
+			host = h["value"].(string)
+		}
+		if h["name"] == ":authority" {
+			authority = h["value"].(string)
+		}
+		if h["name"] == ":scheme" {
+			scheme = h["value"].(string)
+		}
+		if h["name"] == ":path" {
+			path = h["value"].(string)
+		}
+		if h["name"] == ":status" {
+			path = h["value"].(string)
+		}
+	}
+
+	return headers, host, scheme, authority, path, status
+}
+
+func BuildPostParams(rawParams []interface{}) []har.Param {
+	params := make([]har.Param, 0, len(rawParams))
+	for _, param := range rawParams {
+		p := param.(map[string]interface{})
+		name := ""
+		if p["name"] != nil {
+			name = p["name"].(string)
+		}
+		value := ""
+		if p["value"] != nil {
+			value = p["value"].(string)
+		}
+		fileName := ""
+		if p["fileName"] != nil {
+			fileName = p["fileName"].(string)
+		}
+		contentType := ""
+		if p["contentType"] != nil {
+			contentType = p["contentType"].(string)
+		}
+
+		params = append(params, har.Param{
+			Name:        name,
+			Value:       value,
+			Filename:    fileName,
+			ContentType: contentType,
+		})
+	}
+
+	return params
+}
+
+func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error) {
+	reqDetails := request.Payload.(map[string]interface{})["details"].(map[string]interface{})
+
+	headers, host, scheme, authority, path, _ := BuildHeaders(reqDetails["headers"].([]interface{}))
+	cookies := make([]har.Cookie, 0) // BuildCookies(reqDetails["cookies"].([]interface{}))
+
+	postData, _ := reqDetails["postData"].(map[string]interface{})
+	mimeType, _ := postData["mimeType"]
+	if mimeType == nil || len(mimeType.(string)) == 0 {
+		mimeType = "text/html"
+	}
+	text, _ := postData["text"]
+	postDataText := ""
+	if text != nil {
+		postDataText = text.(string)
+	}
+
+	queryString := make([]har.QueryString, 0)
+	for _, _qs := range reqDetails["queryString"].([]interface{}) {
+		qs := _qs.(map[string]interface{})
+		queryString = append(queryString, har.QueryString{
+			Name:  qs["name"].(string),
+			Value: qs["value"].(string),
+		})
+	}
+
+	url := fmt.Sprintf("http://%s%s", host, reqDetails["url"].(string))
+	if strings.HasPrefix(mimeType.(string), "application/grpc") {
+		url = fmt.Sprintf("%s://%s%s", scheme, authority, path)
+	}
+
+	harParams := make([]har.Param, 0)
+	if postData["params"] != nil {
+		harParams = BuildPostParams(postData["params"].([]interface{}))
+	}
+
+	harRequest = &har.Request{
+		Method:      reqDetails["method"].(string),
+		URL:         url,
+		HTTPVersion: reqDetails["httpVersion"].(string),
+		HeadersSize: -1,
+		BodySize:    int64(bytes.NewBufferString(postDataText).Len()),
+		QueryString: queryString,
+		Headers:     headers,
+		Cookies:     cookies,
+		PostData: &har.PostData{
+			MimeType: mimeType.(string),
+			Params:   harParams,
+			Text:     postDataText,
+		},
+	}
+
+	return
+}
+
+func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err error) {
+	resDetails := response.Payload.(map[string]interface{})["details"].(map[string]interface{})
+
+	headers, _, _, _, _, _status := BuildHeaders(resDetails["headers"].([]interface{}))
+	cookies := make([]har.Cookie, 0) // BuildCookies(resDetails["cookies"].([]interface{}))
+
+	content, _ := resDetails["content"].(map[string]interface{})
+	mimeType, _ := content["mimeType"]
+	if mimeType == nil || len(mimeType.(string)) == 0 {
+		mimeType = "text/html"
+	}
+	encoding, _ := content["encoding"]
+	text, _ := content["text"]
+	bodyText := ""
+	if text != nil {
+		bodyText = text.(string)
+	}
+
+	harContent := &har.Content{
+		Encoding: encoding.(string),
+		MimeType: mimeType.(string),
+		Text:     []byte(bodyText),
+		Size:     int64(len(bodyText)),
+	}
+
+	status := int(resDetails["status"].(float64))
+	if strings.HasPrefix(mimeType.(string), "application/grpc") {
+		status, err = strconv.Atoi(_status)
+		if err != nil {
+			tap.SilentError("convert-response-status-for-har", "Failed converting status to int %s (%v,%+v)", err, err, err)
+			return nil, errors.New("failed converting response status to int for HAR")
+		}
+	}
+
+	harResponse = &har.Response{
+		HTTPVersion: resDetails["httpVersion"].(string),
+		Status:      status,
+		StatusText:  resDetails["statusText"].(string),
+		HeadersSize: -1,
+		BodySize:    int64(bytes.NewBufferString(bodyText).Len()),
+		Headers:     headers,
+		Cookies:     cookies,
+		Content:     harContent,
+	}
+	return
+}
+
+func NewEntry(pair *api.RequestResponsePair) (*har.Entry, error) {
+	harRequest, err := NewRequest(&pair.Request)
+	if err != nil {
+		tap.SilentError("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)", err, err, err)
+		return nil, errors.New("failed converting request to HAR")
+	}
+
+	harResponse, err := NewResponse(&pair.Response)
+	if err != nil {
+		fmt.Printf("err: %+v\n", err)
+		tap.SilentError("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)", err, err, err)
+		return nil, errors.New("failed converting response to HAR")
+	}
+
+	totalTime := pair.Response.CaptureTime.Sub(pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
+	if totalTime < 1 {
+		totalTime = 1
+	}
+
+	harEntry := har.Entry{
+		StartedDateTime: pair.Request.CaptureTime,
+		Time:            totalTime,
+		Request:         harRequest,
+		Response:        harResponse,
+		Cache:           &har.Cache{},
+		Timings: &har.Timings{
+			Send:    -1,
+			Wait:    -1,
+			Receive: totalTime,
+		},
+	}
+
+	return &harEntry, nil
+}

ui/src/components/EntryDetailed/EntryViewer.module.sass

@@ -2,7 +2,7 @@
 
 .Entry
     font-family: "Source Sans Pro", Lucida Grande, Tahoma, sans-serif
-    height: 100%
+    height: calc(100% - 70px)
     width: 100%
 
     h3,
@@ -44,6 +44,8 @@
         border-top-right-radius: 0
 
     .body
+        height: 100%
+        overflow-y: auto
         background: $main-background-color
         color: $blue-gray
         border-radius: 4px