diff --git a/api/pkg/api/main.go b/api/pkg/api/main.go index 4df93f500..7cfa1031c 100644 --- a/api/pkg/api/main.go +++ b/api/pkg/api/main.go @@ -145,7 +145,10 @@ func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) { } database.GetEntriesTable().Create(&mizuEntry) - baseEntry := utils.GetResolvedBaseEntry(mizuEntry) + baseEntry := models.BaseEntryDetails{} + if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil { + return + } baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry) broadcastToBrowserClients(baseEntryBytes) } diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index a2139b297..d3c85a794 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -40,10 +40,13 @@ func GetEntries(c *fiber.Ctx) error { utils.ReverseSlice(entries) } - // Convert to base entries - baseEntries := make([]models.BaseEntryDetails, 0, entriesFilter.Limit) - for _, entry := range entries { - baseEntries = append(baseEntries, utils.GetResolvedBaseEntry(entry)) + baseEntries := make([]models.BaseEntryDetails, 0) + for _, data := range entries { + harEntry := models.BaseEntryDetails{} + if err := models.GetEntry(&data, &harEntry); err != nil { + continue + } + baseEntries = append(baseEntries, harEntry) } return c.Status(fiber.StatusOK).JSON(baseEntries) @@ -177,24 +180,31 @@ func GetFullEntries(c *fiber.Ctx) error { } entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo) - return c.Status(fiber.StatusOK).JSON(entriesArray) + result := make([]models.FullEntryDetails, 0) + for _, data := range entriesArray { + harEntry := models.FullEntryDetails{} + if err := models.GetEntry(&data, &harEntry); err != nil { + continue + } + result = append(result, harEntry) + } + + return c.Status(fiber.StatusOK).JSON(result) } func GetEntry(c *fiber.Ctx) error { - var entryData models.EntryData + var entryData models.MizuEntry database.GetEntriesTable(). - Select("entry", "resolvedDestination"). Where(map[string]string{"entryId": c.Params("entryId")}). First(&entryData) - var fullEntry har.Entry - unmarshallErr := json.Unmarshal([]byte(entryData.Entry), &fullEntry) - utils.CheckErr(unmarshallErr) - - if entryData.ResolvedDestination != "" { - fullEntry.Request.URL = utils.SetHostname(fullEntry.Request.URL, entryData.ResolvedDestination) + fullEntry := models.FullEntryDetails{} + if err := models.GetEntry(&entryData, &fullEntry); err != nil { + return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ + "error": true, + "msg": "Can't get entry details", + }) } - return c.Status(fiber.StatusOK).JSON(fullEntry) } diff --git a/api/pkg/database/main.go b/api/pkg/database/main.go index f14734f85..4f144400e 100644 --- a/api/pkg/database/main.go +++ b/api/pkg/database/main.go @@ -1,9 +1,7 @@ package database import ( - "encoding/json" "fmt" - "github.com/google/martian/har" "gorm.io/driver/sqlite" "gorm.io/gorm" "mizuserver/pkg/models" @@ -46,7 +44,8 @@ func initDataBase(databasePath string) *gorm.DB { return temp } -func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []har.Entry { + +func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []models.MizuEntry { order := OrderDesc var entries []models.MizuEntry GetEntriesTable(). @@ -58,21 +57,6 @@ func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []har.Entry { // the entries always order from oldest to newest so we should revers utils.ReverseSlice(entries) } - - entriesArray := make([]har.Entry, 0) - for _, entryData := range entries { - var harEntry har.Entry - _ = json.Unmarshal([]byte(entryData.Entry), &harEntry) - - if entryData.ResolvedSource != "" { - harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: entryData.ResolvedSource}) - } - if entryData.ResolvedDestination != "" { - harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entryData.ResolvedDestination}) - } - - entriesArray = append(entriesArray, harEntry) - } - return entriesArray + return entries } diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index 232a5c1f8..8c0d07e22 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -5,9 +5,18 @@ import ( "github.com/google/martian/har" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/tap" + "mizuserver/pkg/utils" "time" ) +type DataUnmarshaler interface { + UnmarshalData(*MizuEntry) error +} + +func GetEntry(r *MizuEntry, v DataUnmarshaler) error { + return v.UnmarshalData(r) +} + type MizuEntry struct { ID uint `gorm:"primarykey"` CreatedAt time.Time @@ -38,6 +47,61 @@ type BaseEntryDetails struct { IsOutgoing bool `json:"isOutgoing,omitempty"` } +type FullEntryDetails struct { + har.Entry +} + +type FullEntryDetailsExtra struct { + har.Entry +} + +func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error { + entryUrl := entry.Url + service := entry.Service + if entry.ResolvedDestination != "" { + entryUrl = utils.SetHostname(entryUrl, entry.ResolvedDestination) + service = utils.SetHostname(service, entry.ResolvedDestination) + } + bed.Id = entry.EntryId + bed.Url = entryUrl + bed.Service = service + bed.Path = entry.Path + bed.StatusCode = entry.Status + bed.Method = entry.Method + bed.Timestamp = entry.Timestamp + bed.RequestSenderIp = entry.RequestSenderIp + bed.IsOutgoing = entry.IsOutgoing + return nil +} + +func (fed *FullEntryDetails) UnmarshalData(entry *MizuEntry) error { + if err := json.Unmarshal([]byte(entry.Entry), &fed.Entry); err != nil { + return err + } + + if entry.ResolvedDestination != "" { + fed.Entry.Request.URL = utils.SetHostname(fed.Entry.Request.URL, entry.ResolvedDestination) + } + return nil +} + +func (fedex *FullEntryDetailsExtra) UnmarshalData(entry *MizuEntry) error { + if err := json.Unmarshal([]byte(entry.Entry), &fedex.Entry); err != nil { + return err + } + + if entry.ResolvedSource != "" { + fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-source", Value: entry.ResolvedSource}) + } + if entry.ResolvedDestination != "" { + fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entry.ResolvedDestination}) + fedex.Entry.Request.URL = utils.SetHostname(fedex.Entry.Request.URL, entry.ResolvedDestination) + } + return nil +} + + + type EntryData struct { Entry string `json:"entry,omitempty"` ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` @@ -50,7 +114,7 @@ type EntriesFilter struct { } type UploadEntriesRequestBody struct { - Dest string `query:"dest"` + Dest string `query:"dest"` } type HarFetchRequestBody struct { diff --git a/api/pkg/up9/main.go b/api/pkg/up9/main.go index 17f1ab131..3601f5321 100644 --- a/api/pkg/up9/main.go +++ b/api/pkg/up9/main.go @@ -9,6 +9,7 @@ import ( "io/ioutil" "log" "mizuserver/pkg/database" + "mizuserver/pkg/models" "net/http" "net/url" "time" @@ -119,9 +120,18 @@ func UploadEntriesImpl(token string, model string, envPrefix string) { entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo) if len(entriesArray) > 0 { - fmt.Printf("About to upload %v entries\n", len(entriesArray)) - body, jMarshalErr := json.Marshal(entriesArray) + fullEntriesExtra := make([]models.FullEntryDetailsExtra, 0) + for _, data := range entriesArray { + harEntry := models.FullEntryDetailsExtra{} + if err := models.GetEntry(&data, &harEntry); err != nil { + continue + } + fullEntriesExtra = append(fullEntriesExtra, harEntry) + } + fmt.Printf("About to upload %v entries\n", len(fullEntriesExtra)) + + body, jMarshalErr := json.Marshal(fullEntriesExtra) if jMarshalErr != nil { analyzeInformation.Reset() fmt.Println("Stopping analyzing") diff --git a/api/pkg/utils/utils.go b/api/pkg/utils/utils.go index c50f65c9d..b66bb5cb0 100644 --- a/api/pkg/utils/utils.go +++ b/api/pkg/utils/utils.go @@ -1,11 +1,9 @@ package utils import ( - "encoding/json" "fmt" "github.com/gofiber/fiber/v2" "log" - "mizuserver/pkg/models" "net/url" "os" "os/signal" @@ -61,28 +59,3 @@ func SetHostname(address, newHostname string) string { return replacedUrl.String() } - -func GetResolvedBaseEntry(entry models.MizuEntry) models.BaseEntryDetails { - entryUrl := entry.Url - service := entry.Service - if entry.ResolvedDestination != "" { - entryUrl = SetHostname(entryUrl, entry.ResolvedDestination) - service = SetHostname(service, entry.ResolvedDestination) - } - return models.BaseEntryDetails{ - Id: entry.EntryId, - Url: entryUrl, - Service: service, - Path: entry.Path, - StatusCode: entry.Status, - Method: entry.Method, - Timestamp: entry.Timestamp, - RequestSenderIp: entry.RequestSenderIp, - IsOutgoing: entry.IsOutgoing, - } -} - -func GetBytesFromStruct(v interface{}) []byte{ - a, _ := json.Marshal(v) - return a -}