Add requestSenderIP (#39)

api/main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"flag"
-
 	"github.com/gofiber/fiber/v2"
 	"mizuserver/pkg/inserter"
 	"mizuserver/pkg/middleware"
@@ -19,8 +18,7 @@ func main() {
 	app := fiber.New()
 
 	// process to read files / channel and insert to DB
-	go inserter.StartReadingFiles(harOutputChannel, tap.HarOutputDir)
+	go inserter.StartReadingEntries(harOutputChannel, tap.HarOutputDir)
-
 
 	middleware.FiberMiddleware(app) // Register Fiber's middleware for app.
 	app.Static("/", "./site")

api/pkg/controllers/entries_controller.go

@@ -29,7 +29,6 @@ var (
 	}
 )
 
-
 func GetEntries(c *fiber.Ctx) error {
 	entriesFilter := &models.EntriesFilter{}
 
@@ -67,6 +66,7 @@ func GetEntries(c *fiber.Ctx) error {
 			StatusCode:      entry.Status,
 			Method:          entry.Method,
 			Timestamp:       entry.Timestamp,
+			RequestSenderIp: entry.RequestSenderIp,
 		})
 	}
 

api/pkg/inserter/main.go

@@ -9,6 +9,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"mizuserver/pkg/database"
 	"mizuserver/pkg/models"
+	"mizuserver/pkg/tap"
 	"mizuserver/pkg/utils"
 	"net/url"
 	"os"
@@ -17,7 +18,7 @@ import (
 	"time"
 )
 
-func StartReadingFiles(harChannel chan *har.Entry, workingDir *string) {
+func StartReadingEntries(harChannel chan *tap.OutputChannelItem, workingDir *string) {
 	if workingDir != nil && *workingDir != "" {
 		startReadingFiles(*workingDir)
 	} else {
@@ -34,7 +35,7 @@ func startReadingFiles(workingDir string) {
 		dirFiles, _ := dir.Readdir(-1)
 		sort.Sort(utils.ByModTime(dirFiles))
 
-		if len(dirFiles) == 0{
+		if len(dirFiles) == 0 {
 			fmt.Printf("Waiting for new files\n")
 			time.Sleep(3 * time.Second)
 			continue
@@ -50,20 +51,20 @@ func startReadingFiles(workingDir string) {
 
 		for _, entry := range inputHar.Log.Entries {
 			time.Sleep(time.Millisecond * 250)
-			saveHarToDb(*entry, "")
+			saveHarToDb(entry, fileInfo.Name())
 		}
 		rmErr := os.Remove(inputFilePath)
 		utils.CheckErr(rmErr)
 	}
 }
 
-func startReadingChannel(harChannel chan *har.Entry) {
+func startReadingChannel(outputItems chan *tap.OutputChannelItem) {
-	for entry := range harChannel {
+	for item := range outputItems {
-		saveHarToDb(*entry, "")
+		saveHarToDb(item.HarEntry, item.RequestSenderIp)
 	}
 }
 
-func saveHarToDb(entry har.Entry, source string) {
+func saveHarToDb(entry *har.Entry, sender string) {
 	entryBytes, _ := json.Marshal(entry)
 	serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL)
 	entryId := primitive.NewObjectID().Hex()
@@ -75,7 +76,7 @@ func saveHarToDb(entry har.Entry, source string) {
 		Path:            urlPath,
 		Method:          entry.Request.Method,
 		Status:          entry.Response.Status,
-		Source:    source,
+		RequestSenderIp: sender,
 		Timestamp:       entry.StartedDateTime.UnixNano() / int64(time.Millisecond),
 	}
 	database.GetEntriesTable().Create(&mizuEntry)
@@ -87,6 +88,7 @@ func saveHarToDb(entry har.Entry, source string) {
 		Path:            urlPath,
 		StatusCode:      entry.Response.Status,
 		Method:          entry.Request.Method,
+		RequestSenderIp: sender,
 		Timestamp:       entry.StartedDateTime.UnixNano() / int64(time.Millisecond),
 	}
 	baseEntryBytes, _ := json.Marshal(&baseEntry)

api/pkg/models/models.go

@@ -11,7 +11,7 @@ type MizuEntry struct {
 	Url             string `json:"url" gorm:"column:url"`
 	Method          string `json:"method" gorm:"column:method"`
 	Status          int    `json:"status" gorm:"column:status"`
-	Source    string `json:"source" gorm:"column:source"`
+	RequestSenderIp string `json:"requestSenderIp" gorm:"column:requestSenderIp"`
 	Service         string `json:"service" gorm:"column:service"`
 	Timestamp       int64  `json:"timestamp" gorm:"column:timestamp"`
 	Path            string `json:"path" gorm:"column:path"`
@@ -20,6 +20,7 @@ type MizuEntry struct {
 type BaseEntryDetails struct {
 	Id              string `json:"id,omitempty"`
 	Url             string `json:"url,omitempty"`
+	RequestSenderIp string `json:"requestSenderIp,omitempty"`
 	Service         string `json:"service,omitempty"`
 	Path            string `json:"path,omitempty"`
 	StatusCode      int    `json:"statusCode,omitempty"`

api/pkg/tap/grpc_assembler.go

@@ -64,7 +64,7 @@ func (fbs *fragmentsByStream) appendFrame(streamID uint32, frame http2.Frame) {
 func (fbs *fragmentsByStream) pop(streamID uint32) ([]hpack.HeaderField, []byte) {
 	headers := (*fbs)[streamID].headers
 	data := (*fbs)[streamID].data
-	delete((*fbs), streamID)
+	delete(*fbs, streamID)
 
 	return headers, data
 }
@@ -193,7 +193,7 @@ func checkIsHTTP2ServerStream(b *bufio.Reader) (bool, error) {
 
 	// Check server connection preface (a settings frame)
 	frameHeader := http2.FrameHeader{
-		Length:   (uint32(buf[0])<<16 | uint32(buf[1])<<8 | uint32(buf[2])),
+		Length:   uint32(buf[0])<<16 | uint32(buf[1])<<8 | uint32(buf[2]),
 		Type:     http2.FrameType(buf[3]),
 		Flags:    http2.Flags(buf[4]),
 		StreamID: binary.BigEndian.Uint32(buf[5:]) & (1<<31 - 1),

api/pkg/tap/har_writer.go

@@ -22,6 +22,7 @@ type PairChanItem struct {
 	RequestTime     time.Time
 	Response        *http.Response
 	ResponseTime    time.Time
+	RequestSenderIp string
 }
 
 func openNewHarFile(filename string) *HarFile {
@@ -153,27 +154,33 @@ func NewHarWriter(outputDir string, maxEntries int) *HarWriter {
 		OutputDirPath: outputDir,
 		MaxEntries: maxEntries,
 		PairChan: make(chan *PairChanItem),
-		OutChan: make(chan *har.Entry, 1000),
+		OutChan: make(chan *OutputChannelItem, 1000),
 		currentFile: nil,
 		done: make(chan bool),
 	}
 }
 
+type OutputChannelItem struct {
+	HarEntry        *har.Entry
+	RequestSenderIp string
+}
+
 type HarWriter struct {
 	OutputDirPath string
 	MaxEntries int
 	PairChan chan *PairChanItem
-	OutChan chan *har.Entry
+	OutChan chan *OutputChannelItem
 	currentFile *HarFile
 	done chan bool
 }
 
-func (hw *HarWriter) WritePair(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time) {
+func (hw *HarWriter) WritePair(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time, requestSenderIp string) {
 	hw.PairChan <- &PairChanItem{
 		Request:         request,
 		RequestTime:     requestTime,
 		Response:        response,
 		ResponseTime:    responseTime,
+		RequestSenderIp: requestSenderIp,
 	}
 }
 
@@ -202,7 +209,10 @@ func (hw *HarWriter) Start() {
 					hw.closeFile()
 				}
 			} else {
-				hw.OutChan <- harEntry
+				hw.OutChan <- &OutputChannelItem{
+					HarEntry:        harEntry,
+					RequestSenderIp: pair.RequestSenderIp,
+				}
 			}
 		}
 
@@ -229,7 +239,10 @@ func (hw *HarWriter) closeFile() {
 	hw.currentFile = nil
 
 	filename := buildFilename(hw.OutputDirPath, time.Now())
-	os.Rename(tmpFilename, filename)
+	err := os.Rename(tmpFilename, filename)
+	if err != nil {
+		SilentError("Rename-file", "cannot rename file: %s (%v,%+v)\n", err, err, err)
+	}
 }
 
 func buildFilename(dir string, t time.Time) string {

api/pkg/tap/http_matcher.go

@@ -36,8 +36,10 @@ type httpMessage struct {
 	Body            messageBody    `json:"body"`
 	captureTime     time.Time
 	orig            interface {}
+	requestSenderIp string
 }
 
+
 // Key is {client_addr}:{client_port}->{dest_addr}:{dest_port}
 type requestResponseMatcher struct {
 	openMessagesMap cmap.ConcurrentMap
@@ -58,7 +60,7 @@ func (matcher *requestResponseMatcher) registerRequest(ident string, request *ht
 		{Key: "x-up9-destination", Value: split[1] + ":" + split[3]},
 	}
 
-	requestHTTPMessage := requestToMessage(request, captureTime, body, &messageExtraHeaders, isHTTP2)
+	requestHTTPMessage := requestToMessage(request, captureTime, body, &messageExtraHeaders, isHTTP2, split[0])
 
 	if response, found := matcher.openMessagesMap.Pop(key); found {
 		// Type assertion always succeeds because all of the map's values are of httpMessage type
@@ -109,7 +111,7 @@ func (matcher *requestResponseMatcher) preparePair(requestHTTPMessage *httpMessa
 	}
 }
 
-func requestToMessage(request *http.Request, captureTime time.Time, body string, messageExtraHeaders *[]headerKeyVal, isHTTP2 bool) httpMessage {
+func requestToMessage(request *http.Request, captureTime time.Time, body string, messageExtraHeaders *[]headerKeyVal, isHTTP2 bool, requestSenderIp string) httpMessage {
 	messageHeaders := make([]headerKeyVal, 0)
 
 	for key, value := range request.Header {
@@ -138,6 +140,7 @@ func requestToMessage(request *http.Request, captureTime time.Time, body string,
 		Body:            requestBody,
 		captureTime:     captureTime,
 		orig:            request,
+		requestSenderIp: requestSenderIp,
 	}
 }
 

api/pkg/tap/http_reader.go

@@ -34,7 +34,7 @@ func (tid *tcpID) String() string {
 /* httpReader gets reads from a channel of bytes of tcp payload, and parses it into HTTP/1 requests and responses.
  * The payload is written to the channel by a tcpStream object that is dedicated to one tcp connection.
  * An httpReader object is unidirectional: it parses either a client stream or a server stream.
- * Implemets io.Reader interface (Read)
+ * Implements io.Reader interface (Read)
  */
 type httpReader struct {
 	ident         string
@@ -80,13 +80,16 @@ func (h *httpReader) run(wg *sync.WaitGroup) {
 	}
 
 	if h.isHTTP2 {
-		prepareHTTP2Connection(b, h.isClient)
+		err := prepareHTTP2Connection(b, h.isClient)
+		if err != nil {
+			SilentError("HTTP/2-Prepare-Connection-After-Check", "stream %s error: %s (%v,%+v)\n", h.ident, err, err, err)
+		}
 		h.grpcAssembler = createGrpcAssembler(b)
 	}
 
 	for true {
 		if h.isHTTP2 {
-			err := h.handleHTTP2Stream(b)
+			err := h.handleHTTP2Stream()
 			if err == io.EOF || err == io.ErrUnexpectedEOF {
 				break
 			} else if err != nil {
@@ -113,11 +116,11 @@ func (h *httpReader) run(wg *sync.WaitGroup) {
 	}
 }
 
-func (h *httpReader) handleHTTP2Stream(b *bufio.Reader) error {
+func (h *httpReader) handleHTTP2Stream() error {
-	streamID, messageHTTP1, body, error := h.grpcAssembler.readMessage()
+	streamID, messageHTTP1, body, err := h.grpcAssembler.readMessage()
 	h.messageCount++
-	if error != nil {
+	if err != nil {
-		return error
+		return err
 	}
 
 	var reqResPair *envoyMessageWrapper
@@ -138,15 +141,14 @@ func (h *httpReader) handleHTTP2Stream(b *bufio.Reader) error {
 				reqResPair.HttpBufferedTrace.Request.captureTime,
 				reqResPair.HttpBufferedTrace.Response.orig.(*http.Response),
 				reqResPair.HttpBufferedTrace.Response.captureTime,
+				reqResPair.HttpBufferedTrace.Request.requestSenderIp,
 			)
 		} else {
 			jsonStr, err := json.Marshal(reqResPair)
-
-			broadcastReqResPair(jsonStr)
-
 			if err != nil {
 				return err
 			}
+			broadcastReqResPair(jsonStr)
 		}
 	}
 
@@ -167,7 +169,9 @@ func (h *httpReader) handleHTTP1ClientStream(b *bufio.Reader) error {
 	} else if h.hexdump {
 		Info("Body(%d/0x%x)\n%s\n", len(body), len(body), hex.Dump(body))
 	}
-	req.Body.Close()
+	if err := req.Body.Close(); err != nil {
+		SilentError("HTTP-request-body-close", "stream %s Failed to close request body: %s\n", h.ident, err)
+	}
 	encoding := req.Header["Content-Encoding"]
 	bodyStr, err := readBody(body, encoding)
 	if err != nil {
@@ -184,15 +188,14 @@ func (h *httpReader) handleHTTP1ClientStream(b *bufio.Reader) error {
 				reqResPair.HttpBufferedTrace.Request.captureTime,
 				reqResPair.HttpBufferedTrace.Response.orig.(*http.Response),
 				reqResPair.HttpBufferedTrace.Response.captureTime,
+				reqResPair.HttpBufferedTrace.Request.requestSenderIp,
 			)
 		} else {
 			jsonStr, err := json.Marshal(reqResPair)
-
-			broadcastReqResPair(jsonStr)
-
 			if err != nil {
 				SilentError("HTTP-marshal", "stream %s Error convert request response to json: %s\n", h.ident, err)
 			}
+			broadcastReqResPair(jsonStr)
 		}
 	}
 
@@ -226,7 +229,9 @@ func (h *httpReader) handleHTTP1ServerStream(b *bufio.Reader) error {
 	if h.hexdump {
 		Info("Body(%d/0x%x)\n%s\n", len(body), len(body), hex.Dump(body))
 	}
-	res.Body.Close()
+	if err := res.Body.Close(); err != nil {
+		SilentError("HTTP-response-body-close", "HTTP/%s: failed to close body(parsed len:%d): %s\n", h.ident, s, err)
+	}
 	sym := ","
 	if res.ContentLength > 0 && res.ContentLength != int64(s) {
 		sym = "!="
@@ -251,15 +256,14 @@ func (h *httpReader) handleHTTP1ServerStream(b *bufio.Reader) error {
 				reqResPair.HttpBufferedTrace.Request.captureTime,
 				reqResPair.HttpBufferedTrace.Response.orig.(*http.Response),
 				reqResPair.HttpBufferedTrace.Response.captureTime,
+				reqResPair.HttpBufferedTrace.Request.requestSenderIp,
 			)
 		} else {
 			jsonStr, err := json.Marshal(reqResPair)
-
-			broadcastReqResPair(jsonStr)
-
 			if err != nil {
 				SilentError("HTTP-marshal", "stream %s Error convert request response to json: %s\n", h.ident, err)
 			}
+			broadcastReqResPair(jsonStr)
 		}
 	}
 

api/pkg/tap/passive_tapper.go

@@ -29,11 +29,10 @@ import (
 	"github.com/google/gopacket/layers" // pulls in all layers decoders
 	"github.com/google/gopacket/pcap"
 	"github.com/google/gopacket/reassembly"
-	"github.com/google/martian/har"
 )
 
 const AppPortsEnvVar = "APP_PORTS"
-const TapOutPortEnvVar = "WEB_SOCKET_PORT"
+const OutPortEnvVar = "WEB_SOCKET_PORT"
 const maxHTTP2DataLenEnvVar = "HTTP2_DATA_SIZE_LIMIT"
 const hostModeEnvVar = "HOST_MODE"
 // default is 1MB, more than the max size accepted by collector and traffic-dumper
@@ -199,7 +198,7 @@ func (c *Context) GetCaptureInfo() gopacket.CaptureInfo {
 	return c.CaptureInfo
 }
 
-func StartPassiveTapper() chan *har.Entry {
+func StartPassiveTapper() chan *OutputChannelItem {
 	var harWriter *HarWriter
 	if *dumpToHar {
 		harWriter = NewHarWriter(*HarOutputDir, *harEntriesPerFile)
@@ -243,7 +242,7 @@ func startPassiveTapper(harWriter *HarWriter) {
 	}
 	hostAppAddresses = parseHostAppAddresses(*hostAppAddressesString)
 	fmt.Println("Filtering for the following addresses:", hostAppAddresses)
-	tapOutputPort := os.Getenv(TapOutPortEnvVar)
+	tapOutputPort := os.Getenv(OutPortEnvVar)
 	if tapOutputPort == "" {
 		fmt.Println("Received empty/no WEB_SOCKET_PORT env var! falling back to port 8080")
 		tapOutputPort = "8080"
@@ -297,15 +296,15 @@ func startPassiveTapper(harWriter *HarWriter) {
 		// just call pcap.OpenLive if you want a simple handle.
 		inactive, err := pcap.NewInactiveHandle(*iface)
 		if err != nil {
-			log.Fatal("could not create: %v", err)
+			log.Fatalf("could not create: %v", err)
 		}
 		defer inactive.CleanUp()
 		if err = inactive.SetSnapLen(*snaplen); err != nil {
-			log.Fatal("could not set snap length: %v", err)
+			log.Fatalf("could not set snap length: %v", err)
 		} else if err = inactive.SetPromisc(*promisc); err != nil {
-			log.Fatal("could not set promisc mode: %v", err)
+			log.Fatalf("could not set promisc mode: %v", err)
 		} else if err = inactive.SetTimeout(time.Second); err != nil {
-			log.Fatal("could not set timeout: %v", err)
+			log.Fatalf("could not set timeout: %v", err)
 		}
 		if *tstype != "" {
 			if t, err := pcap.TimestampSourceFromString(*tstype); err != nil {
@@ -334,12 +333,12 @@ func startPassiveTapper(harWriter *HarWriter) {
 
 	var dec gopacket.Decoder
 	var ok bool
-	decoder_name := *decoder
+	decoderName := *decoder
-	if decoder_name == "" {
+	if decoderName == "" {
-		decoder_name = fmt.Sprintf("%s", handle.LinkType())
+		decoderName = fmt.Sprintf("%s", handle.LinkType())
 	}
-	if dec, ok = gopacket.DecodersByLayerName[decoder_name]; !ok {
+	if dec, ok = gopacket.DecodersByLayerName[decoderName]; !ok {
-		log.Fatalln("No decoder named", decoder_name)
+		log.Fatalln("No decoder named", decoderName)
 	}
 	source := gopacket.NewPacketSource(handle, dec)
 	source.Lazy = *lazy
@@ -526,7 +525,7 @@ func startPassiveTapper(harWriter *HarWriter) {
 	fmt.Printf(" overlap packets:\t%d\n", stats.overlapPackets)
 	fmt.Printf(" overlap bytes:\t\t%d\n", stats.overlapBytes)
 	fmt.Printf("Errors: %d\n", nErrors)
-	for e, _ := range errorsMap {
+	for e := range errorsMap {
 		fmt.Printf(" %s:\t\t%d\n", e, errorsMap[e])
 	}
 }