Marshal and unmarshal *http.Request, *http.Response pairs

agent/pkg/api/main.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"mizuserver/pkg/database"
 	"mizuserver/pkg/holder"
+	"net/http"
 	"net/url"
 	"os"
 	"path"
@@ -111,13 +112,18 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem) {
 
 	for item := range outputItems {
 		fmt.Printf("item: %+v\n", item)
+		var req *http.Request
+		marshedReq, _ := json.Marshal(item.Data.Request.Orig)
+		json.Unmarshal(marshedReq, &req)
+		var res *http.Response
+		marshedRes, _ := json.Marshal(item.Data.Response.Orig)
+		json.Unmarshal(marshedRes, &res)
 		// NOTE: With this call, the incoming data is sent to the last WebSocket (that the web UI communicates).
-		handleItem(item)
-		// if harEntry, err := models.NewEntry(item.Data.Request.Orig.(*http.Request), item.Data.Request.CaptureTime, item.Data.Response.Orig.(*http.Response), item.Data.Response.CaptureTime); err == nil {
-		// 	saveHarToDb(harEntry, item.ConnectionInfo)
-		// } else {
-		// 	rlog.Errorf("Error when creating HTTP entry")
-		// }
+		if harEntry, err := models.NewEntry(req, item.Data.Request.CaptureTime, res, item.Data.Response.CaptureTime); err == nil {
+			saveHarToDb(harEntry, item.ConnectionInfo)
+		} else {
+			rlog.Errorf("Error when creating HTTP entry")
+		}
 	}
 }
 
@@ -128,15 +134,6 @@ func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) {
 	}
 }
 
-func handleItem(item *tapApi.OutputChannelItem) {
-	baseEntry := models.BaseEntryDetails{
-		RequestSenderIp: item.ConnectionInfo.ClientIP,
-	}
-	baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry)
-	// NOTE: This is where it's sent to the last WebSocket
-	BroadcastToBrowserClients(baseEntryBytes)
-}
-
 func saveHarToDb(entry *har.Entry, connectionInfo *tapApi.ConnectionInfo) {
 	entryBytes, _ := json.Marshal(entry)
 	serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL)

agent/pkg/api/socket_server_handlers.go

@@ -3,12 +3,13 @@ package api
 import (
 	"encoding/json"
 	"fmt"
-	tapApi "github.com/up9inc/mizu/tap/api"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/providers"
 	"mizuserver/pkg/up9"
 	"sync"
 
+	tapApi "github.com/up9inc/mizu/tap/api"
+
 	"github.com/romana/rlog"
 	"github.com/up9inc/mizu/shared"
 )
@@ -71,6 +72,7 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 			if err != nil {
 				rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
 			} else {
+				// NOTE: This is where the message comes back from the intermediate WebSocket to code.
 				h.SocketOutChannel <- tappedEntryMessage.Data
 			}
 		case shared.WebSocketMessageTypeUpdateStatus:

agent/pkg/models/models.go

@@ -5,12 +5,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	tapApi "github.com/up9inc/mizu/tap/api"
 	"io/ioutil"
 	"net/http"
 	"strconv"
 	"strings"
 
+	tapApi "github.com/up9inc/mizu/tap/api"
+
 	"mizuserver/pkg/rules"
 	"mizuserver/pkg/utils"
 	"time"

tap/extensions/http/go.mod

@@ -3,6 +3,7 @@ module github.com/up9inc/mizu/tap/extensions/http
 go 1.16
 
 require (
+	github.com/google/martian v2.1.0+incompatible // indirect
 	github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
 	github.com/up9inc/mizu/tap/api v0.0.0
 	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7

tap/extensions/http/go.sum

@@ -1,10 +1,18 @@
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=
 github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c=
+golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

tap/extensions/http/structs.go

@@ -1,7 +1,13 @@
 package main
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
+	"net/http"
+
+	"github.com/google/martian/har"
+	"github.com/romana/rlog"
 )
 
 type HTTPPayload struct {
@@ -14,12 +20,21 @@ type HTTPPayloader interface {
 }
 
 func (h HTTPPayload) MarshalJSON() ([]byte, error) {
-	// TODO: Implement JSON marshaling for HTTP request and response
 	switch h.Type {
 	case "http_request":
-		return []byte("{\"val\": \"" + h.Type + "\"}"), nil
+		harRequest, err := har.NewRequest(h.Data.(*http.Request), false)
+		if err != nil {
+			rlog.Debugf("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)", err, err, err)
+			return nil, errors.New("Failed converting request to HAR")
+		}
+		return json.Marshal(harRequest)
 	case "http_response":
-		return []byte("{\"val\": \"" + h.Type + "\"}"), nil
+		harResponse, err := har.NewResponse(h.Data.(*http.Response), true)
+		if err != nil {
+			rlog.Debugf("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)", err, err, err)
+			return nil, errors.New("Failed converting response to HAR")
+		}
+		return json.Marshal(harResponse)
 	default:
 		panic(fmt.Sprintf("HTTP payload cannot be marshaled: %s\n", h.Type))
 	}