From 461bcf9f2454704f91193615a41fb6bfd5658d44 Mon Sep 17 00:00:00 2001 From: "M. Mert Yildiran" Date: Fri, 20 Aug 2021 03:00:55 +0300 Subject: [PATCH] Marshal and unmarshal `*http.Request`, `*http.Response` pairs --- agent/pkg/api/main.go | 27 +++++++++++-------------- agent/pkg/api/socket_server_handlers.go | 4 +++- agent/pkg/models/models.go | 3 ++- tap/extensions/http/go.mod | 1 + tap/extensions/http/go.sum | 8 ++++++++ tap/extensions/http/structs.go | 21 ++++++++++++++++--- 6 files changed, 44 insertions(+), 20 deletions(-) diff --git a/agent/pkg/api/main.go b/agent/pkg/api/main.go index 2c3b47c9b..e4d5f5bd0 100644 --- a/agent/pkg/api/main.go +++ b/agent/pkg/api/main.go @@ -7,6 +7,7 @@ import ( "fmt" "mizuserver/pkg/database" "mizuserver/pkg/holder" + "net/http" "net/url" "os" "path" @@ -111,13 +112,18 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem) { for item := range outputItems { fmt.Printf("item: %+v\n", item) + var req *http.Request + marshedReq, _ := json.Marshal(item.Data.Request.Orig) + json.Unmarshal(marshedReq, &req) + var res *http.Response + marshedRes, _ := json.Marshal(item.Data.Response.Orig) + json.Unmarshal(marshedRes, &res) // NOTE: With this call, the incoming data is sent to the last WebSocket (that the web UI communicates). - handleItem(item) - // if harEntry, err := models.NewEntry(item.Data.Request.Orig.(*http.Request), item.Data.Request.CaptureTime, item.Data.Response.Orig.(*http.Response), item.Data.Response.CaptureTime); err == nil { - // saveHarToDb(harEntry, item.ConnectionInfo) - // } else { - // rlog.Errorf("Error when creating HTTP entry") - // } + if harEntry, err := models.NewEntry(req, item.Data.Request.CaptureTime, res, item.Data.Response.CaptureTime); err == nil { + saveHarToDb(harEntry, item.ConnectionInfo) + } else { + rlog.Errorf("Error when creating HTTP entry") + } } } @@ -128,15 +134,6 @@ func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) { } } -func handleItem(item *tapApi.OutputChannelItem) { - baseEntry := models.BaseEntryDetails{ - RequestSenderIp: item.ConnectionInfo.ClientIP, - } - baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry) - // NOTE: This is where it's sent to the last WebSocket - BroadcastToBrowserClients(baseEntryBytes) -} - func saveHarToDb(entry *har.Entry, connectionInfo *tapApi.ConnectionInfo) { entryBytes, _ := json.Marshal(entry) serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL) diff --git a/agent/pkg/api/socket_server_handlers.go b/agent/pkg/api/socket_server_handlers.go index 7afe1c1c6..35b6ef0aa 100644 --- a/agent/pkg/api/socket_server_handlers.go +++ b/agent/pkg/api/socket_server_handlers.go @@ -3,12 +3,13 @@ package api import ( "encoding/json" "fmt" - tapApi "github.com/up9inc/mizu/tap/api" "mizuserver/pkg/models" "mizuserver/pkg/providers" "mizuserver/pkg/up9" "sync" + tapApi "github.com/up9inc/mizu/tap/api" + "github.com/romana/rlog" "github.com/up9inc/mizu/shared" ) @@ -71,6 +72,7 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) { if err != nil { rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err) } else { + // NOTE: This is where the message comes back from the intermediate WebSocket to code. h.SocketOutChannel <- tappedEntryMessage.Data } case shared.WebSocketMessageTypeUpdateStatus: diff --git a/agent/pkg/models/models.go b/agent/pkg/models/models.go index 922027ead..d443e7f5f 100644 --- a/agent/pkg/models/models.go +++ b/agent/pkg/models/models.go @@ -5,12 +5,13 @@ import ( "encoding/json" "errors" "fmt" - tapApi "github.com/up9inc/mizu/tap/api" "io/ioutil" "net/http" "strconv" "strings" + tapApi "github.com/up9inc/mizu/tap/api" + "mizuserver/pkg/rules" "mizuserver/pkg/utils" "time" diff --git a/tap/extensions/http/go.mod b/tap/extensions/http/go.mod index 8f94c6a90..a2105f6eb 100644 --- a/tap/extensions/http/go.mod +++ b/tap/extensions/http/go.mod @@ -3,6 +3,7 @@ module github.com/up9inc/mizu/tap/extensions/http go 1.16 require ( + github.com/google/martian v2.1.0+incompatible // indirect github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 github.com/up9inc/mizu/tap/api v0.0.0 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 diff --git a/tap/extensions/http/go.sum b/tap/extensions/http/go.sum index 786bf6d01..ebfc4bcd6 100644 --- a/tap/extensions/http/go.sum +++ b/tap/extensions/http/go.sum @@ -1,10 +1,18 @@ +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI= github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/tap/extensions/http/structs.go b/tap/extensions/http/structs.go index 1a3f022bc..05be81fc8 100644 --- a/tap/extensions/http/structs.go +++ b/tap/extensions/http/structs.go @@ -1,7 +1,13 @@ package main import ( + "encoding/json" + "errors" "fmt" + "net/http" + + "github.com/google/martian/har" + "github.com/romana/rlog" ) type HTTPPayload struct { @@ -14,12 +20,21 @@ type HTTPPayloader interface { } func (h HTTPPayload) MarshalJSON() ([]byte, error) { - // TODO: Implement JSON marshaling for HTTP request and response switch h.Type { case "http_request": - return []byte("{\"val\": \"" + h.Type + "\"}"), nil + harRequest, err := har.NewRequest(h.Data.(*http.Request), false) + if err != nil { + rlog.Debugf("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)", err, err, err) + return nil, errors.New("Failed converting request to HAR") + } + return json.Marshal(harRequest) case "http_response": - return []byte("{\"val\": \"" + h.Type + "\"}"), nil + harResponse, err := har.NewResponse(h.Data.(*http.Response), true) + if err != nil { + rlog.Debugf("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)", err, err, err) + return nil, errors.New("Failed converting response to HAR") + } + return json.Marshal(harResponse) default: panic(fmt.Sprintf("HTTP payload cannot be marshaled: %s\n", h.Type)) }