Better creation of entries from mizu entry (resolving inside) (#98)

* no message
* no message
* no message
* no message
* no message
* no message

api/pkg/api/main.go

@@ -145,7 +145,10 @@ func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
 	}
 	database.GetEntriesTable().Create(&mizuEntry)
 
-	baseEntry := utils.GetResolvedBaseEntry(mizuEntry)
+	baseEntry := models.BaseEntryDetails{}
+	if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil {
+		return
+	}
 	baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry)
 	broadcastToBrowserClients(baseEntryBytes)
 }

api/pkg/controllers/entries_controller.go

@@ -40,10 +40,13 @@ func GetEntries(c *fiber.Ctx) error {
 		utils.ReverseSlice(entries)
 	}
 
-	// Convert to base entries
+	baseEntries := make([]models.BaseEntryDetails, 0)
-	baseEntries := make([]models.BaseEntryDetails, 0, entriesFilter.Limit)
+	for _, data := range entries {
-	for _, entry := range entries {
+		harEntry := models.BaseEntryDetails{}
-		baseEntries = append(baseEntries, utils.GetResolvedBaseEntry(entry))
+		if err := models.GetEntry(&data, &harEntry); err != nil {
+			continue
+		}
+		baseEntries = append(baseEntries, harEntry)
 	}
 
 	return c.Status(fiber.StatusOK).JSON(baseEntries)
@@ -177,24 +180,31 @@ func GetFullEntries(c *fiber.Ctx) error {
 	}
 
 	entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo)
-	return c.Status(fiber.StatusOK).JSON(entriesArray)
+	result := make([]models.FullEntryDetails, 0)
+	for _, data := range entriesArray {
+		harEntry := models.FullEntryDetails{}
+		if err := models.GetEntry(&data, &harEntry); err != nil {
+			continue
+		}
+		result = append(result, harEntry)
+	}
+
+	return c.Status(fiber.StatusOK).JSON(result)
 }
 
 func GetEntry(c *fiber.Ctx) error {
-	var entryData models.EntryData
+	var entryData models.MizuEntry
 	database.GetEntriesTable().
-		Select("entry", "resolvedDestination").
 		Where(map[string]string{"entryId": c.Params("entryId")}).
 		First(&entryData)
 
-	var fullEntry har.Entry
+	fullEntry := models.FullEntryDetails{}
-	unmarshallErr := json.Unmarshal([]byte(entryData.Entry), &fullEntry)
+	if err := models.GetEntry(&entryData, &fullEntry); err != nil {
-	utils.CheckErr(unmarshallErr)
+		return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
-
+			"error": true,
-	if entryData.ResolvedDestination != "" {
+			"msg":   "Can't get entry details",
-		fullEntry.Request.URL = utils.SetHostname(fullEntry.Request.URL, entryData.ResolvedDestination)
+		})
 	}
-
 	return c.Status(fiber.StatusOK).JSON(fullEntry)
 }
 

api/pkg/database/main.go

@@ -1,9 +1,7 @@
 package database
 
 import (
-	"encoding/json"
 	"fmt"
-	"github.com/google/martian/har"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 	"mizuserver/pkg/models"
@@ -46,7 +44,8 @@ func initDataBase(databasePath string) *gorm.DB {
 	return temp
 }
 
-func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []har.Entry {
+
+func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []models.MizuEntry {
 	order := OrderDesc
 	var entries []models.MizuEntry
 	GetEntriesTable().
@@ -58,21 +57,6 @@ func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []har.Entry {
 		// the entries always order from oldest to newest so we should revers
 		utils.ReverseSlice(entries)
 	}
-
+	return entries
-	entriesArray := make([]har.Entry, 0)
-	for _, entryData := range entries {
-		var harEntry har.Entry
-		_ = json.Unmarshal([]byte(entryData.Entry), &harEntry)
-
-		if entryData.ResolvedSource != "" {
-			harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: entryData.ResolvedSource})
-		}
-		if entryData.ResolvedDestination != "" {
-			harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entryData.ResolvedDestination})
-		}
-
-		entriesArray = append(entriesArray, harEntry)
-	}
-	return entriesArray
 }
 

api/pkg/models/models.go

@@ -5,9 +5,18 @@ import (
 	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"
+	"mizuserver/pkg/utils"
 	"time"
 )
 
+type DataUnmarshaler interface {
+	UnmarshalData(*MizuEntry) error
+}
+
+func GetEntry(r *MizuEntry, v DataUnmarshaler) error {
+	return v.UnmarshalData(r)
+}
+
 type MizuEntry struct {
 	ID                  uint `gorm:"primarykey"`
 	CreatedAt           time.Time
@@ -38,6 +47,61 @@ type BaseEntryDetails struct {
 	IsOutgoing      bool   `json:"isOutgoing,omitempty"`
 }
 
+type FullEntryDetails struct {
+	har.Entry
+}
+
+type FullEntryDetailsExtra struct {
+	har.Entry
+}
+
+func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error {
+	entryUrl := entry.Url
+	service := entry.Service
+	if entry.ResolvedDestination != "" {
+		entryUrl = utils.SetHostname(entryUrl, entry.ResolvedDestination)
+		service = utils.SetHostname(service, entry.ResolvedDestination)
+	}
+	bed.Id = entry.EntryId
+	bed.Url = entryUrl
+	bed.Service = service
+	bed.Path = entry.Path
+	bed.StatusCode = entry.Status
+	bed.Method = entry.Method
+	bed.Timestamp = entry.Timestamp
+	bed.RequestSenderIp = entry.RequestSenderIp
+	bed.IsOutgoing = entry.IsOutgoing
+	return nil
+}
+
+func (fed *FullEntryDetails) UnmarshalData(entry *MizuEntry) error {
+	if err := json.Unmarshal([]byte(entry.Entry), &fed.Entry); err != nil {
+		return err
+	}
+
+	if entry.ResolvedDestination != "" {
+		fed.Entry.Request.URL = utils.SetHostname(fed.Entry.Request.URL, entry.ResolvedDestination)
+	}
+	return nil
+}
+
+func (fedex *FullEntryDetailsExtra) UnmarshalData(entry *MizuEntry) error {
+	if err := json.Unmarshal([]byte(entry.Entry), &fedex.Entry); err != nil {
+		return err
+	}
+
+	if entry.ResolvedSource != "" {
+		fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-source", Value: entry.ResolvedSource})
+	}
+	if entry.ResolvedDestination != "" {
+		fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entry.ResolvedDestination})
+		fedex.Entry.Request.URL = utils.SetHostname(fedex.Entry.Request.URL, entry.ResolvedDestination)
+	}
+	return nil
+}
+
+
+
 type EntryData struct {
 	Entry               string `json:"entry,omitempty"`
 	ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
@@ -50,7 +114,7 @@ type EntriesFilter struct {
 }
 
 type UploadEntriesRequestBody struct {
-	Dest  string `query:"dest"`
+	Dest string `query:"dest"`
 }
 
 type HarFetchRequestBody struct {

api/pkg/up9/main.go

@@ -9,6 +9,7 @@ import (
 	"io/ioutil"
 	"log"
 	"mizuserver/pkg/database"
+	"mizuserver/pkg/models"
 	"net/http"
 	"net/url"
 	"time"
@@ -119,9 +120,18 @@ func UploadEntriesImpl(token string, model string, envPrefix string) {
 		entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo)
 
 		if len(entriesArray) > 0 {
-			fmt.Printf("About to upload %v entries\n", len(entriesArray))
 
-			body, jMarshalErr := json.Marshal(entriesArray)
+			fullEntriesExtra := make([]models.FullEntryDetailsExtra, 0)
+			for _, data := range entriesArray {
+				harEntry := models.FullEntryDetailsExtra{}
+				if err := models.GetEntry(&data, &harEntry); err != nil {
+					continue
+				}
+				fullEntriesExtra = append(fullEntriesExtra, harEntry)
+			}
+			fmt.Printf("About to upload %v entries\n", len(fullEntriesExtra))
+
+			body, jMarshalErr := json.Marshal(fullEntriesExtra)
 			if jMarshalErr != nil {
 				analyzeInformation.Reset()
 				fmt.Println("Stopping analyzing")

api/pkg/utils/utils.go

@@ -1,11 +1,9 @@
 package utils
 
 import (
-	"encoding/json"
 	"fmt"
 	"github.com/gofiber/fiber/v2"
 	"log"
-	"mizuserver/pkg/models"
 	"net/url"
 	"os"
 	"os/signal"
@@ -61,28 +59,3 @@ func SetHostname(address, newHostname string) string {
 	return replacedUrl.String()
 
 }
-
-func GetResolvedBaseEntry(entry models.MizuEntry) models.BaseEntryDetails {
-	entryUrl := entry.Url
-	service := entry.Service
-	if entry.ResolvedDestination != "" {
-		entryUrl = SetHostname(entryUrl, entry.ResolvedDestination)
-		service = SetHostname(service, entry.ResolvedDestination)
-	}
-	return models.BaseEntryDetails{
-		Id:              entry.EntryId,
-		Url:             entryUrl,
-		Service:         service,
-		Path:            entry.Path,
-		StatusCode:      entry.Status,
-		Method:          entry.Method,
-		Timestamp:       entry.Timestamp,
-		RequestSenderIp: entry.RequestSenderIp,
-		IsOutgoing:      entry.IsOutgoing,
-	}
-}
-
-func GetBytesFromStruct(v interface{}) []byte{
-	a, _ := json.Marshal(v)
-	return a
-}