diff --git a/agent/pkg/controllers/entries_controller.go b/agent/pkg/controllers/entries_controller.go index ce639eb26..f4a225aa0 100644 --- a/agent/pkg/controllers/entries_controller.go +++ b/agent/pkg/controllers/entries_controller.go @@ -48,7 +48,7 @@ func GetEntries(c *gin.Context) { Find(&entries) if len(entries) > 0 && order == database.OrderDesc { - // the entries always order from oldest to newest so we should revers + // the entries always order from oldest to newest - we should reverse utils.ReverseSlice(entries) } @@ -95,7 +95,7 @@ func GetHARs(c *gin.Context) { Find(&entries) if len(entries) > 0 { - // the entries always order from oldest to newest so we should revers + // the entries always order from oldest to newest - we should reverse utils.ReverseSlice(entries) } @@ -113,7 +113,7 @@ func GetHARs(c *gin.Context) { if sourceOfEntry != "" { // naively assumes the proper service source is http sourceOfEntry = fmt.Sprintf("http://%s", sourceOfEntry) - //replace / from the file name cause they end up creating a corrupted folder + //replace / from the file name because they end up creating a corrupted folder fileName = fmt.Sprintf("%s.har", strings.ReplaceAll(sourceOfEntry, "/", "_")) } else { fileName = "unknown_source.har" @@ -202,15 +202,21 @@ func GetFullEntries(c *gin.Context) { timestampTo = entriesFilter.To } - entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo) - result := make([]models.FullEntryDetails, 0) + entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo, nil) + + result := make([]har.Entry, 0) for _, data := range entriesArray { - harEntry := models.FullEntryDetails{} - if err := models.GetEntry(&data, &harEntry); err != nil { + var pair tapApi.RequestResponsePair + if err := json.Unmarshal([]byte(data.Entry), &pair); err != nil { continue } - result = append(result, harEntry) + harEntry, err := utils.NewEntry(&pair) + if err != nil { + continue + } + result = append(result, *harEntry) } + c.JSON(http.StatusOK, result) } @@ -220,22 +226,6 @@ func GetEntry(c *gin.Context) { Where(map[string]string{"entryId": c.Param("entryId")}). First(&entryData) - fullEntry := models.FullEntryDetails{} - if err := models.GetEntry(&entryData, &fullEntry); err != nil { - c.JSON(http.StatusInternalServerError, map[string]interface{}{ - "error": true, - "msg": "Can't get entry details", - }) - } - - // FIXME: Fix the part below - // fullEntryWithPolicy := models.FullEntryWithPolicy{} - // if err := models.GetEntry(&entryData, &fullEntryWithPolicy); err != nil { - // c.JSON(http.StatusInternalServerError, map[string]interface{}{ - // "error": true, - // "msg": "Can't get entry details", - // }) - // } extension := extensionsMap[entryData.ProtocolName] protocol, representation, bodySize, _ := extension.Dissector.Represent(&entryData) c.JSON(http.StatusOK, tapApi.MizuEntryWrapper{ diff --git a/agent/pkg/database/main.go b/agent/pkg/database/main.go index f6dfe402e..63fa8b1ce 100644 --- a/agent/pkg/database/main.go +++ b/agent/pkg/database/main.go @@ -57,10 +57,16 @@ func initDataBase(databasePath string) *gorm.DB { return temp } -func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []tapApi.MizuEntry { +func GetEntriesFromDb(timestampFrom int64, timestampTo int64, protocolName *string) []tapApi.MizuEntry { order := OrderDesc + protocolNameCondition := "1 = 1" + if protocolName != nil { + protocolNameCondition = fmt.Sprintf("protocolKey = '%s'", *protocolName) + } + var entries []tapApi.MizuEntry GetEntriesTable(). + Where(protocolNameCondition). Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)). Order(fmt.Sprintf("timestamp %s", order)). Find(&entries) diff --git a/agent/pkg/models/models.go b/agent/pkg/models/models.go index 8a92fc64a..e97069168 100644 --- a/agent/pkg/models/models.go +++ b/agent/pkg/models/models.go @@ -2,9 +2,7 @@ package models import ( "encoding/json" - tapApi "github.com/up9inc/mizu/tap/api" - "mizuserver/pkg/rules" "mizuserver/pkg/utils" @@ -17,47 +15,14 @@ func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error { return v.UnmarshalData(r) } -func NewApplicableRules(status bool, latency int64, number int) tapApi.ApplicableRules { - ar := tapApi.ApplicableRules{} - ar.Status = status - ar.Latency = latency - ar.NumberOfRules = number - return ar -} - -type FullEntryDetails struct { - har.Entry -} - -type FullEntryDetailsExtra struct { - har.Entry -} - -func (fed *FullEntryDetails) UnmarshalData(entry *tapApi.MizuEntry) error { - if err := json.Unmarshal([]byte(entry.Entry), &fed.Entry); err != nil { - return err - } - - if entry.ResolvedDestination != "" { - fed.Entry.Request.URL = utils.SetHostname(fed.Entry.Request.URL, entry.ResolvedDestination) - } - return nil -} - -func (fedex *FullEntryDetailsExtra) UnmarshalData(entry *tapApi.MizuEntry) error { - if err := json.Unmarshal([]byte(entry.Entry), &fedex.Entry); err != nil { - return err - } - - if entry.ResolvedSource != "" { - fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-source", Value: entry.ResolvedSource}) - } - if entry.ResolvedDestination != "" { - fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entry.ResolvedDestination}) - fedex.Entry.Request.URL = utils.SetHostname(fedex.Entry.Request.URL, entry.ResolvedDestination) - } - return nil -} +// TODO: until we fixed the Rules feature +//func NewApplicableRules(status bool, latency int64, number int) tapApi.ApplicableRules { +// ar := tapApi.ApplicableRules{} +// ar.Status = status +// ar.Latency = latency +// ar.NumberOfRules = number +// return ar +//} type EntriesFilter struct { Limit int `form:"limit" validate:"required,min=1,max=200"` @@ -147,9 +112,15 @@ type FullEntryWithPolicy struct { } func (fewp *FullEntryWithPolicy) UnmarshalData(entry *tapApi.MizuEntry) error { - if err := json.Unmarshal([]byte(entry.Entry), &fewp.Entry); err != nil { + var pair tapApi.RequestResponsePair + if err := json.Unmarshal([]byte(entry.Entry), &pair); err != nil { return err } + harEntry, err := utils.NewEntry(&pair) + if err != nil { + return err + } + fewp.Entry = *harEntry _, resultPolicyToSend := rules.MatchRequestPolicy(fewp.Entry, entry.Service) fewp.RulesMatched = resultPolicyToSend @@ -157,9 +128,10 @@ func (fewp *FullEntryWithPolicy) UnmarshalData(entry *tapApi.MizuEntry) error { return nil } -func RunValidationRulesState(harEntry har.Entry, service string) tapApi.ApplicableRules { - numberOfRules, resultPolicyToSend := rules.MatchRequestPolicy(harEntry, service) - statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend, numberOfRules) - ar := NewApplicableRules(statusPolicyToSend, latency, numberOfRules) - return ar -} +// TODO: until we fixed the Rules feature +//func RunValidationRulesState(harEntry har.Entry, service string) tapApi.ApplicableRules { +// numberOfRules, resultPolicyToSend := rules.MatchRequestPolicy(harEntry, service) +// statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend, numberOfRules) +// ar := NewApplicableRules(statusPolicyToSend, latency, numberOfRules) +// return ar +//} diff --git a/agent/pkg/up9/main.go b/agent/pkg/up9/main.go index b932b92af..bfdd7d12d 100644 --- a/agent/pkg/up9/main.go +++ b/agent/pkg/up9/main.go @@ -5,12 +5,14 @@ import ( "compress/zlib" "encoding/json" "fmt" + "github.com/google/martian/har" "github.com/romana/rlog" "github.com/up9inc/mizu/shared" + tapApi "github.com/up9inc/mizu/tap/api" "io/ioutil" "log" "mizuserver/pkg/database" - "mizuserver/pkg/models" + "mizuserver/pkg/utils" "net/http" "net/url" "strings" @@ -129,21 +131,33 @@ func UploadEntriesImpl(token string, model string, envPrefix string, sleepInterv for { timestampTo := time.Now().UnixNano() / int64(time.Millisecond) rlog.Infof("Getting entries from %v, to %v\n", timestampFrom, timestampTo) - entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo) + protocolFilter := "http" + entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo, &protocolFilter) if len(entriesArray) > 0 { - - fullEntriesExtra := make([]models.FullEntryDetailsExtra, 0) + result := make([]har.Entry, 0) for _, data := range entriesArray { - harEntry := models.FullEntryDetailsExtra{} - if err := models.GetEntry(&data, &harEntry); err != nil { + var pair tapApi.RequestResponsePair + if err := json.Unmarshal([]byte(data.Entry), &pair); err != nil { continue } - fullEntriesExtra = append(fullEntriesExtra, harEntry) + harEntry, err := utils.NewEntry(&pair) + if err != nil { + continue + } + if data.ResolvedSource != "" { + harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: data.ResolvedSource}) + } + if data.ResolvedDestination != "" { + harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: data.ResolvedDestination}) + harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, data.ResolvedDestination) + } + result = append(result, *harEntry) } - rlog.Infof("About to upload %v entries\n", len(fullEntriesExtra)) - body, jMarshalErr := json.Marshal(fullEntriesExtra) + rlog.Infof("About to upload %v entries\n", len(result)) + + body, jMarshalErr := json.Marshal(result) if jMarshalErr != nil { analyzeInformation.Reset() rlog.Infof("Stopping analyzing") diff --git a/agent/pkg/utils/har.go b/agent/pkg/utils/har.go new file mode 100644 index 000000000..5cfdad8af --- /dev/null +++ b/agent/pkg/utils/har.go @@ -0,0 +1,259 @@ +package utils + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/google/martian/har" + "github.com/up9inc/mizu/tap" + "github.com/up9inc/mizu/tap/api" +) + + +// Keep it because we might want cookies in the future +//func BuildCookies(rawCookies []interface{}) []har.Cookie { +// cookies := make([]har.Cookie, 0, len(rawCookies)) +// +// for _, cookie := range rawCookies { +// c := cookie.(map[string]interface{}) +// expiresStr := "" +// if c["expires"] != nil { +// expiresStr = c["expires"].(string) +// } +// expires, _ := time.Parse(time.RFC3339, expiresStr) +// httpOnly := false +// if c["httponly"] != nil { +// httpOnly, _ = strconv.ParseBool(c["httponly"].(string)) +// } +// secure := false +// if c["secure"] != nil { +// secure, _ = strconv.ParseBool(c["secure"].(string)) +// } +// path := "" +// if c["path"] != nil { +// path = c["path"].(string) +// } +// domain := "" +// if c["domain"] != nil { +// domain = c["domain"].(string) +// } +// +// cookies = append(cookies, har.Cookie{ +// Name: c["name"].(string), +// Value: c["value"].(string), +// Path: path, +// Domain: domain, +// HTTPOnly: httpOnly, +// Secure: secure, +// Expires: expires, +// Expires8601: expiresStr, +// }) +// } +// +// return cookies +//} + +func BuildHeaders(rawHeaders []interface{}) ([]har.Header, string, string, string, string, string) { + var host, scheme, authority, path, status string + headers := make([]har.Header, 0, len(rawHeaders)) + + for _, header := range rawHeaders { + h := header.(map[string]interface{}) + + headers = append(headers, har.Header{ + Name: h["name"].(string), + Value: h["value"].(string), + }) + + if h["name"] == "Host" { + host = h["value"].(string) + } + if h["name"] == ":authority" { + authority = h["value"].(string) + } + if h["name"] == ":scheme" { + scheme = h["value"].(string) + } + if h["name"] == ":path" { + path = h["value"].(string) + } + if h["name"] == ":status" { + path = h["value"].(string) + } + } + + return headers, host, scheme, authority, path, status +} + +func BuildPostParams(rawParams []interface{}) []har.Param { + params := make([]har.Param, 0, len(rawParams)) + for _, param := range rawParams { + p := param.(map[string]interface{}) + name := "" + if p["name"] != nil { + name = p["name"].(string) + } + value := "" + if p["value"] != nil { + value = p["value"].(string) + } + fileName := "" + if p["fileName"] != nil { + fileName = p["fileName"].(string) + } + contentType := "" + if p["contentType"] != nil { + contentType = p["contentType"].(string) + } + + params = append(params, har.Param{ + Name: name, + Value: value, + Filename: fileName, + ContentType: contentType, + }) + } + + return params +} + +func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error) { + reqDetails := request.Payload.(map[string]interface{})["details"].(map[string]interface{}) + + headers, host, scheme, authority, path, _ := BuildHeaders(reqDetails["headers"].([]interface{})) + cookies := make([]har.Cookie, 0) // BuildCookies(reqDetails["cookies"].([]interface{})) + + postData, _ := reqDetails["postData"].(map[string]interface{}) + mimeType, _ := postData["mimeType"] + if mimeType == nil || len(mimeType.(string)) == 0 { + mimeType = "text/html" + } + text, _ := postData["text"] + postDataText := "" + if text != nil { + postDataText = text.(string) + } + + queryString := make([]har.QueryString, 0) + for _, _qs := range reqDetails["queryString"].([]interface{}) { + qs := _qs.(map[string]interface{}) + queryString = append(queryString, har.QueryString{ + Name: qs["name"].(string), + Value: qs["value"].(string), + }) + } + + url := fmt.Sprintf("http://%s%s", host, reqDetails["url"].(string)) + if strings.HasPrefix(mimeType.(string), "application/grpc") { + url = fmt.Sprintf("%s://%s%s", scheme, authority, path) + } + + harParams := make([]har.Param, 0) + if postData["params"] != nil { + harParams = BuildPostParams(postData["params"].([]interface{})) + } + + harRequest = &har.Request{ + Method: reqDetails["method"].(string), + URL: url, + HTTPVersion: reqDetails["httpVersion"].(string), + HeadersSize: -1, + BodySize: int64(bytes.NewBufferString(postDataText).Len()), + QueryString: queryString, + Headers: headers, + Cookies: cookies, + PostData: &har.PostData{ + MimeType: mimeType.(string), + Params: harParams, + Text: postDataText, + }, + } + + return +} + +func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err error) { + resDetails := response.Payload.(map[string]interface{})["details"].(map[string]interface{}) + + headers, _, _, _, _, _status := BuildHeaders(resDetails["headers"].([]interface{})) + cookies := make([]har.Cookie, 0) // BuildCookies(resDetails["cookies"].([]interface{})) + + content, _ := resDetails["content"].(map[string]interface{}) + mimeType, _ := content["mimeType"] + if mimeType == nil || len(mimeType.(string)) == 0 { + mimeType = "text/html" + } + encoding, _ := content["encoding"] + text, _ := content["text"] + bodyText := "" + if text != nil { + bodyText = text.(string) + } + + harContent := &har.Content{ + Encoding: encoding.(string), + MimeType: mimeType.(string), + Text: []byte(bodyText), + Size: int64(len(bodyText)), + } + + status := int(resDetails["status"].(float64)) + if strings.HasPrefix(mimeType.(string), "application/grpc") { + status, err = strconv.Atoi(_status) + if err != nil { + tap.SilentError("convert-response-status-for-har", "Failed converting status to int %s (%v,%+v)", err, err, err) + return nil, errors.New("failed converting response status to int for HAR") + } + } + + harResponse = &har.Response{ + HTTPVersion: resDetails["httpVersion"].(string), + Status: status, + StatusText: resDetails["statusText"].(string), + HeadersSize: -1, + BodySize: int64(bytes.NewBufferString(bodyText).Len()), + Headers: headers, + Cookies: cookies, + Content: harContent, + } + return +} + +func NewEntry(pair *api.RequestResponsePair) (*har.Entry, error) { + harRequest, err := NewRequest(&pair.Request) + if err != nil { + tap.SilentError("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)", err, err, err) + return nil, errors.New("failed converting request to HAR") + } + + harResponse, err := NewResponse(&pair.Response) + if err != nil { + fmt.Printf("err: %+v\n", err) + tap.SilentError("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)", err, err, err) + return nil, errors.New("failed converting response to HAR") + } + + totalTime := pair.Response.CaptureTime.Sub(pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds() + if totalTime < 1 { + totalTime = 1 + } + + harEntry := har.Entry{ + StartedDateTime: pair.Request.CaptureTime, + Time: totalTime, + Request: harRequest, + Response: harResponse, + Cache: &har.Cache{}, + Timings: &har.Timings{ + Send: -1, + Wait: -1, + Receive: totalTime, + }, + } + + return &harEntry, nil +} diff --git a/ui/src/components/EntryDetailed/EntryViewer.module.sass b/ui/src/components/EntryDetailed/EntryViewer.module.sass index fd8c882da..740a4c417 100644 --- a/ui/src/components/EntryDetailed/EntryViewer.module.sass +++ b/ui/src/components/EntryDetailed/EntryViewer.module.sass @@ -2,7 +2,7 @@ .Entry font-family: "Source Sans Pro", Lucida Grande, Tahoma, sans-serif - height: 100% + height: calc(100% - 70px) width: 100% h3, @@ -44,6 +44,8 @@ border-top-right-radius: 0 .body + height: 100% + overflow-y: auto background: $main-background-color color: $blue-gray border-radius: 4px