github/kubeshark
1
0
Fork 0
mirror of https://github.com/kubeshark/kubeshark.git synced 2025-08-31 10:11:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Better creation of entries from mizu entry (resolving inside) (#98)

Browse Source 
* no message
* no message
* no message
* no message
* no message
* no message
This commit is contained in:
gadotroee
2021-07-07 17:11:57 +03:00
committed by GitHub
parent 12d873d344
commit 7abf8b83e3
6 changed files with 108 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
api/pkg/api/main.go
View File

@@ -145,7 +145,10 @@ func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
}
database.GetEntriesTable().Create(&mizuEntry)
baseEntry := utils.GetResolvedBaseEntry(mizuEntry)
baseEntry := models.BaseEntryDetails{}
if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil {
return
}
baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry)
broadcastToBrowserClients(baseEntryBytes)
}

38
api/pkg/controllers/entries_controller.go
View File

@@ -40,10 +40,13 @@ func GetEntries(c *fiber.Ctx) error {
utils.ReverseSlice(entries)
}
// Convert to base entries
baseEntries := make([]models.BaseEntryDetails, 0, entriesFilter.Limit)
for _, entry := range entries {
baseEntries = append(baseEntries, utils.GetResolvedBaseEntry(entry))
baseEntries := make([]models.BaseEntryDetails, 0)
for _, data := range entries {
harEntry := models.BaseEntryDetails{}
if err := models.GetEntry(&data, &harEntry); err != nil {
continue
}
baseEntries = append(baseEntries, harEntry)
}
return c.Status(fiber.StatusOK).JSON(baseEntries)
@@ -177,24 +180,31 @@ func GetFullEntries(c *fiber.Ctx) error {
}
entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo)
return c.Status(fiber.StatusOK).JSON(entriesArray)
result := make([]models.FullEntryDetails, 0)
for _, data := range entriesArray {
harEntry := models.FullEntryDetails{}
if err := models.GetEntry(&data, &harEntry); err != nil {
continue
}
result = append(result, harEntry)
}
return c.Status(fiber.StatusOK).JSON(result)
}
func GetEntry(c *fiber.Ctx) error {
var entryData models.EntryData
var entryData models.MizuEntry
database.GetEntriesTable().
Select("entry", "resolvedDestination").
Where(map[string]string{"entryId": c.Params("entryId")}).
First(&entryData)
var fullEntry har.Entry
unmarshallErr := json.Unmarshal([]byte(entryData.Entry), &fullEntry)
utils.CheckErr(unmarshallErr)
if entryData.ResolvedDestination != "" {
fullEntry.Request.URL = utils.SetHostname(fullEntry.Request.URL, entryData.ResolvedDestination)
fullEntry := models.FullEntryDetails{}
if err := models.GetEntry(&entryData, &fullEntry); err != nil {
return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
"error": true,
"msg": "Can't get entry details",
})
}
return c.Status(fiber.StatusOK).JSON(fullEntry)
}

22
api/pkg/database/main.go
View File

@@ -1,9 +1,7 @@
package database
import (
"encoding/json"
"fmt"
"github.com/google/martian/har"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"mizuserver/pkg/models"
@@ -46,7 +44,8 @@ func initDataBase(databasePath string) *gorm.DB {
return temp
}
func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []har.Entry {
func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []models.MizuEntry {
order := OrderDesc
var entries []models.MizuEntry
GetEntriesTable().
@@ -58,21 +57,6 @@ func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []har.Entry {
// the entries always order from oldest to newest so we should revers
utils.ReverseSlice(entries)
}
entriesArray := make([]har.Entry, 0)
for _, entryData := range entries {
var harEntry har.Entry
_ = json.Unmarshal([]byte(entryData.Entry), &harEntry)
if entryData.ResolvedSource != "" {
harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: entryData.ResolvedSource})
}
if entryData.ResolvedDestination != "" {
harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entryData.ResolvedDestination})
}
entriesArray = append(entriesArray, harEntry)
}
return entriesArray
return entries
}

66
api/pkg/models/models.go
View File

@@ -5,9 +5,18 @@ import (
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
"mizuserver/pkg/utils"
"time"
)
type DataUnmarshaler interface {
UnmarshalData(*MizuEntry) error
}
func GetEntry(r *MizuEntry, v DataUnmarshaler) error {
return v.UnmarshalData(r)
}
type MizuEntry struct {
ID uint `gorm:"primarykey"`
CreatedAt time.Time
@@ -38,6 +47,61 @@ type BaseEntryDetails struct {
IsOutgoing bool `json:"isOutgoing,omitempty"`
}
type FullEntryDetails struct {
har.Entry
}
type FullEntryDetailsExtra struct {
har.Entry
}
func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error {
entryUrl := entry.Url
service := entry.Service
if entry.ResolvedDestination != "" {
entryUrl = utils.SetHostname(entryUrl, entry.ResolvedDestination)
service = utils.SetHostname(service, entry.ResolvedDestination)
}
bed.Id = entry.EntryId
bed.Url = entryUrl
bed.Service = service
bed.Path = entry.Path
bed.StatusCode = entry.Status
bed.Method = entry.Method
bed.Timestamp = entry.Timestamp
bed.RequestSenderIp = entry.RequestSenderIp
bed.IsOutgoing = entry.IsOutgoing
return nil
}
func (fed *FullEntryDetails) UnmarshalData(entry *MizuEntry) error {
if err := json.Unmarshal([]byte(entry.Entry), &fed.Entry); err != nil {
return err
}
if entry.ResolvedDestination != "" {
fed.Entry.Request.URL = utils.SetHostname(fed.Entry.Request.URL, entry.ResolvedDestination)
}
return nil
}
func (fedex *FullEntryDetailsExtra) UnmarshalData(entry *MizuEntry) error {
if err := json.Unmarshal([]byte(entry.Entry), &fedex.Entry); err != nil {
return err
}
if entry.ResolvedSource != "" {
fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-source", Value: entry.ResolvedSource})
}
if entry.ResolvedDestination != "" {
fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entry.ResolvedDestination})
fedex.Entry.Request.URL = utils.SetHostname(fedex.Entry.Request.URL, entry.ResolvedDestination)
}
return nil
}
type EntryData struct {
Entry string `json:"entry,omitempty"`
ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
@@ -50,7 +114,7 @@ type EntriesFilter struct {
}
type UploadEntriesRequestBody struct {
Dest string `query:"dest"`
Dest string `query:"dest"`
}
type HarFetchRequestBody struct {

14
api/pkg/up9/main.go
View File

@@ -9,6 +9,7 @@ import (
"io/ioutil"
"log"
"mizuserver/pkg/database"
"mizuserver/pkg/models"
"net/http"
"net/url"
"time"
@@ -119,9 +120,18 @@ func UploadEntriesImpl(token string, model string, envPrefix string) {
entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo)
if len(entriesArray) > 0 {
fmt.Printf("About to upload %v entries\n", len(entriesArray))
body, jMarshalErr := json.Marshal(entriesArray)
fullEntriesExtra := make([]models.FullEntryDetailsExtra, 0)
for _, data := range entriesArray {
harEntry := models.FullEntryDetailsExtra{}
if err := models.GetEntry(&data, &harEntry); err != nil {
continue
}
fullEntriesExtra = append(fullEntriesExtra, harEntry)
}
fmt.Printf("About to upload %v entries\n", len(fullEntriesExtra))
body, jMarshalErr := json.Marshal(fullEntriesExtra)
if jMarshalErr != nil {
analyzeInformation.Reset()
fmt.Println("Stopping analyzing")

27
api/pkg/utils/utils.go
View File

@@ -1,11 +1,9 @@
package utils
import (
"encoding/json"
"fmt"
"github.com/gofiber/fiber/v2"
"log"
"mizuserver/pkg/models"
"net/url"
"os"
"os/signal"
@@ -61,28 +59,3 @@ func SetHostname(address, newHostname string) string {
return replacedUrl.String()
}
func GetResolvedBaseEntry(entry models.MizuEntry) models.BaseEntryDetails {
entryUrl := entry.Url
service := entry.Service
if entry.ResolvedDestination != "" {
entryUrl = SetHostname(entryUrl, entry.ResolvedDestination)
service = SetHostname(service, entry.ResolvedDestination)
}
return models.BaseEntryDetails{
Id: entry.EntryId,
Url: entryUrl,
Service: service,
Path: entry.Path,
StatusCode: entry.Status,
Method: entry.Method,
Timestamp: entry.Timestamp,
RequestSenderIp: entry.RequestSenderIp,
IsOutgoing: entry.IsOutgoing,
}
}
func GetBytesFromStruct(v interface{}) []byte{
a, _ := json.Marshal(v)
return a
}