Better creation of entries from mizu entry (resolving inside) (#98)

* no message
* no message
* no message
* no message
* no message
* no message

api/pkg/api/main.go

@@ -5,10 +5,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"mizuserver/pkg/database"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/resolver"
-	"mizuserver/pkg/utils"
 	"net/url"
 	"os"
 	"path"
@@ -19,6 +15,11 @@ import (
 	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/tap"
 	"go.mongodb.org/mongo-driver/bson/primitive"
+
+	"mizuserver/pkg/database"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/resolver"
+	"mizuserver/pkg/utils"
 )
 
 var k8sResolver *resolver.Resolver
@@ -84,7 +85,14 @@ func startReadingFiles(workingDir string) {
 
 		for _, entry := range inputHar.Log.Entries {
 			time.Sleep(time.Millisecond * 250)
-			saveHarToDb(entry, fileInfo.Name(), false)
+			connectionInfo := &tap.ConnectionInfo{
+				ClientIP: fileInfo.Name(),
+				ClientPort: "",
+				ServerIP: "",
+				ServerPort: "",
+				IsOutgoing: false,
+			}
+			saveHarToDb(entry, connectionInfo)
 		}
 		rmErr := os.Remove(inputFilePath)
 		utils.CheckErr(rmErr)
@@ -97,7 +105,7 @@ func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) {
 	}
 
 	for item := range outputItems {
-		saveHarToDb(item.HarEntry, item.ConnectionInfo.ClientIP, item.ConnectionInfo.IsOutgoing)
+		saveHarToDb(item.HarEntry, item.ConnectionInfo)
 	}
 }
 
@@ -109,17 +117,17 @@ func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) {
 }
 
 
-func saveHarToDb(entry *har.Entry, sender string, isOutgoing bool) {
+func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
 	entryBytes, _ := json.Marshal(entry)
-	serviceName, urlPath, serviceHostName := getServiceNameFromUrl(entry.Request.URL)
+	serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL)
 	entryId := primitive.NewObjectID().Hex()
 	var (
 		resolvedSource      string
 		resolvedDestination string
 	)
 	if k8sResolver != nil {
-		resolvedSource = k8sResolver.Resolve(sender)
-		resolvedDestination = k8sResolver.Resolve(serviceHostName)
+		resolvedSource = k8sResolver.Resolve(connectionInfo.ClientIP)
+		resolvedDestination = k8sResolver.Resolve(fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort))
 	}
 	mizuEntry := models.MizuEntry{
 		EntryId:             entryId,
@@ -129,23 +137,26 @@ func saveHarToDb(entry *har.Entry, sender string, isOutgoing bool) {
 		Path:                urlPath,
 		Method:              entry.Request.Method,
 		Status:              entry.Response.Status,
-		RequestSenderIp:     sender,
+		RequestSenderIp:     connectionInfo.ClientIP,
 		Timestamp:           entry.StartedDateTime.UnixNano() / int64(time.Millisecond),
 		ResolvedSource:      resolvedSource,
 		ResolvedDestination: resolvedDestination,
-		IsOutgoing:          isOutgoing,
+		IsOutgoing:          connectionInfo.IsOutgoing,
 	}
 	database.GetEntriesTable().Create(&mizuEntry)
 
-	baseEntry := utils.GetResolvedBaseEntry(mizuEntry)
+	baseEntry := models.BaseEntryDetails{}
+	if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil {
+		return
+	}
 	baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry)
 	broadcastToBrowserClients(baseEntryBytes)
 }
 
-func getServiceNameFromUrl(inputUrl string) (string, string, string) {
+func getServiceNameFromUrl(inputUrl string) (string, string) {
 	parsed, err := url.Parse(inputUrl)
 	utils.CheckErr(err)
-	return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host), parsed.Path, parsed.Host
+	return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host), parsed.Path
 }
 
 func CheckIsServiceIP(address string) bool {

api/pkg/controllers/entries_controller.go

@@ -40,10 +40,13 @@ func GetEntries(c *fiber.Ctx) error {
 		utils.ReverseSlice(entries)
 	}
 
-	// Convert to base entries
-	baseEntries := make([]models.BaseEntryDetails, 0, entriesFilter.Limit)
-	for _, entry := range entries {
-		baseEntries = append(baseEntries, utils.GetResolvedBaseEntry(entry))
+	baseEntries := make([]models.BaseEntryDetails, 0)
+	for _, data := range entries {
+		harEntry := models.BaseEntryDetails{}
+		if err := models.GetEntry(&data, &harEntry); err != nil {
+			continue
+		}
+		baseEntries = append(baseEntries, harEntry)
 	}
 
 	return c.Status(fiber.StatusOK).JSON(baseEntries)
@@ -93,13 +96,16 @@ func GetHARs(c *fiber.Ctx) error {
 			harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entryData.ResolvedDestination)
 		}
 
+		var fileName string
 		sourceOfEntry := entryData.ResolvedSource
 		if sourceOfEntry != "" {
 			// naively assumes the proper service source is http
 			sourceOfEntry = fmt.Sprintf("http://%s", sourceOfEntry)
+			//replace / from the file name cause they end up creating a corrupted folder
+			fileName = fmt.Sprintf("%s.har", strings.ReplaceAll(sourceOfEntry, "/", "_"))
+		} else {
+			fileName = "unknown_source.har"
 		}
-		//replace / from the file name cause they end up creating a corrupted folder
-		fileName := fmt.Sprintf("%s.har", strings.ReplaceAll(sourceOfEntry, "/", "_"))
 		if harOfSource, ok := harsObject[fileName]; ok {
 			harOfSource.Log.Entries = append(harOfSource.Log.Entries, &harEntry)
 		} else {
@@ -119,7 +125,7 @@ func GetHARs(c *fiber.Ctx) error {
 			}
 			// leave undefined when no source is present, otherwise modeler assumes source is empty string ""
 			if sourceOfEntry != "" {
-				harsObject[fileName].Log.Creator.Source = sourceOfEntry
+				harsObject[fileName].Log.Creator.Source = &sourceOfEntry
 			}
 		}
 	}
@@ -174,24 +180,31 @@ func GetFullEntries(c *fiber.Ctx) error {
 	}
 
 	entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo)
-	return c.Status(fiber.StatusOK).JSON(entriesArray)
+	result := make([]models.FullEntryDetails, 0)
+	for _, data := range entriesArray {
+		harEntry := models.FullEntryDetails{}
+		if err := models.GetEntry(&data, &harEntry); err != nil {
+			continue
+		}
+		result = append(result, harEntry)
+	}
+
+	return c.Status(fiber.StatusOK).JSON(result)
 }
 
 func GetEntry(c *fiber.Ctx) error {
-	var entryData models.EntryData
+	var entryData models.MizuEntry
 	database.GetEntriesTable().
-		Select("entry", "resolvedDestination").
 		Where(map[string]string{"entryId": c.Params("entryId")}).
 		First(&entryData)
 
-	var fullEntry har.Entry
-	unmarshallErr := json.Unmarshal([]byte(entryData.Entry), &fullEntry)
-	utils.CheckErr(unmarshallErr)
-
-	if entryData.ResolvedDestination != "" {
-		fullEntry.Request.URL = utils.SetHostname(fullEntry.Request.URL, entryData.ResolvedDestination)
+	fullEntry := models.FullEntryDetails{}
+	if err := models.GetEntry(&entryData, &fullEntry); err != nil {
+		return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
+			"error": true,
+			"msg":   "Can't get entry details",
+		})
 	}
-
 	return c.Status(fiber.StatusOK).JSON(fullEntry)
 }
 

api/pkg/database/main.go

@@ -1,9 +1,7 @@
 package database
 
 import (
-	"encoding/json"
 	"fmt"
-	"github.com/google/martian/har"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 	"mizuserver/pkg/models"
@@ -46,7 +44,8 @@ func initDataBase(databasePath string) *gorm.DB {
 	return temp
 }
 
-func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []har.Entry {
+
+func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []models.MizuEntry {
 	order := OrderDesc
 	var entries []models.MizuEntry
 	GetEntriesTable().
@@ -58,21 +57,6 @@ func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []har.Entry {
 		// the entries always order from oldest to newest so we should revers
 		utils.ReverseSlice(entries)
 	}
-
-	entriesArray := make([]har.Entry, 0)
-	for _, entryData := range entries {
-		var harEntry har.Entry
-		_ = json.Unmarshal([]byte(entryData.Entry), &harEntry)
-
-		if entryData.ResolvedSource != "" {
-			harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: entryData.ResolvedSource})
-		}
-		if entryData.ResolvedDestination != "" {
-			harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entryData.ResolvedDestination})
-		}
-
-		entriesArray = append(entriesArray, harEntry)
-	}
-	return entriesArray
+	return entries
 }
 

api/pkg/models/models.go

@@ -5,9 +5,18 @@ import (
 	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"
+	"mizuserver/pkg/utils"
 	"time"
 )
 
+type DataUnmarshaler interface {
+	UnmarshalData(*MizuEntry) error
+}
+
+func GetEntry(r *MizuEntry, v DataUnmarshaler) error {
+	return v.UnmarshalData(r)
+}
+
 type MizuEntry struct {
 	ID                  uint `gorm:"primarykey"`
 	CreatedAt           time.Time
@@ -38,6 +47,61 @@ type BaseEntryDetails struct {
 	IsOutgoing      bool   `json:"isOutgoing,omitempty"`
 }
 
+type FullEntryDetails struct {
+	har.Entry
+}
+
+type FullEntryDetailsExtra struct {
+	har.Entry
+}
+
+func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error {
+	entryUrl := entry.Url
+	service := entry.Service
+	if entry.ResolvedDestination != "" {
+		entryUrl = utils.SetHostname(entryUrl, entry.ResolvedDestination)
+		service = utils.SetHostname(service, entry.ResolvedDestination)
+	}
+	bed.Id = entry.EntryId
+	bed.Url = entryUrl
+	bed.Service = service
+	bed.Path = entry.Path
+	bed.StatusCode = entry.Status
+	bed.Method = entry.Method
+	bed.Timestamp = entry.Timestamp
+	bed.RequestSenderIp = entry.RequestSenderIp
+	bed.IsOutgoing = entry.IsOutgoing
+	return nil
+}
+
+func (fed *FullEntryDetails) UnmarshalData(entry *MizuEntry) error {
+	if err := json.Unmarshal([]byte(entry.Entry), &fed.Entry); err != nil {
+		return err
+	}
+
+	if entry.ResolvedDestination != "" {
+		fed.Entry.Request.URL = utils.SetHostname(fed.Entry.Request.URL, entry.ResolvedDestination)
+	}
+	return nil
+}
+
+func (fedex *FullEntryDetailsExtra) UnmarshalData(entry *MizuEntry) error {
+	if err := json.Unmarshal([]byte(entry.Entry), &fedex.Entry); err != nil {
+		return err
+	}
+
+	if entry.ResolvedSource != "" {
+		fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-source", Value: entry.ResolvedSource})
+	}
+	if entry.ResolvedDestination != "" {
+		fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entry.ResolvedDestination})
+		fedex.Entry.Request.URL = utils.SetHostname(fedex.Entry.Request.URL, entry.ResolvedDestination)
+	}
+	return nil
+}
+
+
+
 type EntryData struct {
 	Entry               string `json:"entry,omitempty"`
 	ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
@@ -50,7 +114,7 @@ type EntriesFilter struct {
 }
 
 type UploadEntriesRequestBody struct {
-	Dest  string `query:"dest"`
+	Dest string `query:"dest"`
 }
 
 type HarFetchRequestBody struct {
@@ -105,5 +169,5 @@ type ExtendedLog struct {
 
 type ExtendedCreator struct {
 	*har.Creator
-	Source string `json:"_source"`
+	Source *string `json:"_source"`
 }

api/pkg/up9/main.go

@@ -9,6 +9,7 @@ import (
 	"io/ioutil"
 	"log"
 	"mizuserver/pkg/database"
+	"mizuserver/pkg/models"
 	"net/http"
 	"net/url"
 	"time"
@@ -119,9 +120,18 @@ func UploadEntriesImpl(token string, model string, envPrefix string) {
 		entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo)
 
 		if len(entriesArray) > 0 {
-			fmt.Printf("About to upload %v entries\n", len(entriesArray))
 
-			body, jMarshalErr := json.Marshal(entriesArray)
+			fullEntriesExtra := make([]models.FullEntryDetailsExtra, 0)
+			for _, data := range entriesArray {
+				harEntry := models.FullEntryDetailsExtra{}
+				if err := models.GetEntry(&data, &harEntry); err != nil {
+					continue
+				}
+				fullEntriesExtra = append(fullEntriesExtra, harEntry)
+			}
+			fmt.Printf("About to upload %v entries\n", len(fullEntriesExtra))
+
+			body, jMarshalErr := json.Marshal(fullEntriesExtra)
 			if jMarshalErr != nil {
 				analyzeInformation.Reset()
 				fmt.Println("Stopping analyzing")

api/pkg/utils/utils.go

@@ -1,11 +1,9 @@
 package utils
 
 import (
-	"encoding/json"
 	"fmt"
 	"github.com/gofiber/fiber/v2"
 	"log"
-	"mizuserver/pkg/models"
 	"net/url"
 	"os"
 	"os/signal"
@@ -61,28 +59,3 @@ func SetHostname(address, newHostname string) string {
 	return replacedUrl.String()
 
 }
-
-func GetResolvedBaseEntry(entry models.MizuEntry) models.BaseEntryDetails {
-	entryUrl := entry.Url
-	service := entry.Service
-	if entry.ResolvedDestination != "" {
-		entryUrl = SetHostname(entryUrl, entry.ResolvedDestination)
-		service = SetHostname(service, entry.ResolvedDestination)
-	}
-	return models.BaseEntryDetails{
-		Id:              entry.EntryId,
-		Url:             entryUrl,
-		Service:         service,
-		Path:            entry.Path,
-		StatusCode:      entry.Status,
-		Method:          entry.Method,
-		Timestamp:       entry.Timestamp,
-		RequestSenderIp: entry.RequestSenderIp,
-		IsOutgoing:      entry.IsOutgoing,
-	}
-}
-
-func GetBytesFromStruct(v interface{}) []byte{
-	a, _ := json.Marshal(v)
-	return a
-}