Reduce delay between tap and UI - Skip dump to file (#26)

* Pass HARs between tap and api via channel.

* Fixed make docker commad.

* Various fixes.

* Added .DS_Store to .gitignore.

* Parse flags in Mizu main instead of in tap_output.go.

* Use channel to pass HAR by default instead of files.

.gitignore

@@ -19,3 +19,6 @@ build
 
 # Build directories
 build
+
+# Mac OS
+.DS_Store

Makefile

@@ -38,7 +38,7 @@ tap: ## build tap binary
 
 docker: ## build Docker image 
 	@(echo "building docker image" )
-	docker build -t ${DOCKER_IMG}:${DOCKER_TAG} api
+	docker build -t ${DOCKER_IMG}:${DOCKER_TAG} .
 	docker images ${DOCKER_IMG}
 
 publish: ## build and publish Mizu docker image & CLI

api/main.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"flag"
+
 	"github.com/gofiber/fiber/v2"
 	"mizuserver/pkg/inserter"
 	"mizuserver/pkg/middleware"
@@ -10,12 +12,14 @@ import (
 )
 
 func main() {
+	flag.Parse()
 
-	go tap.StartPassiveTapper()
+	harOutputChannel := tap.StartPassiveTapper()
 
 	app := fiber.New()
 
-	go inserter.StartReadingFiles(*tap.HarOutputDir)  // process to read files and insert to DB
+	// process to read files / channel and insert to DB
+	go inserter.StartReadingFiles(harOutputChannel, tap.HarOutputDir)
 
 
 	middleware.FiberMiddleware(app) // Register Fiber's middleware for app.

api/pkg/inserter/main.go

@@ -17,7 +17,15 @@ import (
 	"time"
 )
 
-func StartReadingFiles(workingDir string) {
+func StartReadingFiles(harChannel chan *har.Entry, workingDir *string) {
+	if workingDir != nil && *workingDir != "" {
+		startReadingFiles(*workingDir)
+	} else {
+		startReadingSChan(harChannel)
+	}
+}
+
+func startReadingFiles(workingDir string) {
 	err := os.MkdirAll(workingDir, os.ModePerm)
 	utils.CheckErr(err)
 
@@ -49,6 +57,12 @@ func StartReadingFiles(workingDir string) {
 	}
 }
 
+func startReadingSChan(harChannel chan *har.Entry) {
+	for entry := range harChannel {
+		SaveHarToDb(*entry, "")
+	}
+}
+
 func SaveHarToDb(entry har.Entry, source string) {
 	entryBytes, _ := json.Marshal(entry)
 	serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL)

api/pkg/tap/har_writer.go

@@ -2,6 +2,7 @@ package tap
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"os"
@@ -38,15 +39,15 @@ type HarFile struct {
 	entryCount int
 }
 
-func (f *HarFile) WriteEntry(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time) {
+func NewEntry(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time) (*har.Entry, error) {
 	// TODO: quick fix until TRA-3212 is implemented  
 	if request.URL == nil || request.Method == "" {
-		return
+		return nil, errors.New("Invalid request")
 	}
 	harRequest, err := har.NewRequest(request, true)
 	if err != nil {
 		SilentError("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)\n", err, err, err)
-		return
+		return nil, errors.New("Failed converting request to HAR")
 	}
 
 	// Martian copies http.Request.URL.String() to har.Request.URL.
@@ -56,7 +57,7 @@ func (f *HarFile) WriteEntry(request *http.Request, requestTime time.Time, respo
 	harResponse, err := har.NewResponse(response, true)
 	if err != nil {
 		SilentError("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)\n", err, err, err)
-		return
+		return nil, errors.New("Failed converting response to HAR")
 	}
 
 	totalTime := responseTime.Sub(requestTime).Round(time.Millisecond).Milliseconds()
@@ -77,6 +78,10 @@ func (f *HarFile) WriteEntry(request *http.Request, requestTime time.Time, respo
 		},
 	}
 
+	return &harEntry, nil
+}
+
+func (f *HarFile) WriteEntry(harEntry *har.Entry) {
 	harEntryJson, err := json.Marshal(harEntry)
 	if err != nil {
 		SilentError("har-entry-marshal", "Failed converting har entry object to JSON%s (%v,%+v)\n", err, err, err)
@@ -131,6 +136,7 @@ func NewHarWriter(outputDir string, maxEntries int) *HarWriter {
 		OutputDirPath: outputDir,
 		MaxEntries: maxEntries,
 		PairChan: make(chan *PairChanItem),
+		OutChan: make(chan *har.Entry, 1000),
 		currentFile: nil,
 		done: make(chan bool),
 	}
@@ -140,6 +146,7 @@ type HarWriter struct {
 	OutputDirPath string
 	MaxEntries int
 	PairChan chan *PairChanItem
+	OutChan chan *har.Entry
 	currentFile *HarFile
 	done chan bool
 }
@@ -154,21 +161,32 @@ func (hw *HarWriter) WritePair(request *http.Request, requestTime time.Time, res
 }
 
 func (hw *HarWriter) Start() {
+	if hw.OutputDirPath != "" {
 		if err := os.MkdirAll(hw.OutputDirPath, os.ModePerm); err != nil {
 			panic(fmt.Sprintf("Failed to create output directory: %s (%v,%+v)", err, err, err))
 		}
+	}
 
 	go func() {
 		for pair := range hw.PairChan {
+			harEntry, err := NewEntry(pair.Request, pair.RequestTime, pair.Response, pair.ResponseTime)
+			if err != nil {
+				continue
+			}
+
+			if hw.OutputDirPath != "" {
 				if hw.currentFile == nil {
 					hw.openNewFile()
 				}
 
-			hw.currentFile.WriteEntry(pair.Request, pair.RequestTime, pair.Response, pair.ResponseTime)
+				hw.currentFile.WriteEntry(harEntry)
 
 				if hw.currentFile.GetEntryCount() >= hw.MaxEntries {
 					hw.closeFile()
 				}
+			} else {
+				hw.OutChan <- harEntry
+			}
 		}
 
 		if hw.currentFile != nil {

api/pkg/tap/passive_tapper.go

@@ -29,6 +29,7 @@ import (
 	"github.com/google/gopacket/layers" // pulls in all layers decoders
 	"github.com/google/gopacket/pcap"
 	"github.com/google/gopacket/reassembly"
+	"github.com/google/martian/har"
 )
 
 const AppPortsEnvVar = "APP_PORTS"
@@ -96,7 +97,7 @@ var memprofile = flag.String("memprofile", "", "Write memory profile")
 
 // output
 var dumpToHar = flag.Bool("hardump", false, "Dump traffic to har files")
-var HarOutputDir = flag.String("hardir", "output", "Directory in which to store output har files")
+var HarOutputDir = flag.String("hardir", "", "Directory in which to store output har files")
 var harEntriesPerFile = flag.Int("harentriesperfile", 200, "Number of max number of har entries to store in each file")
 
 var reqResMatcher = createResponseRequestMatcher()  // global
@@ -198,7 +199,22 @@ func (c *Context) GetCaptureInfo() gopacket.CaptureInfo {
 	return c.CaptureInfo
 }
 
-func StartPassiveTapper() {
+func StartPassiveTapper() chan *har.Entry {
+	var harWriter *HarWriter
+	if *dumpToHar {
+		harWriter = NewHarWriter(*HarOutputDir, *harEntriesPerFile)
+	}
+
+	go startPassiveTapper(harWriter)
+
+	if harWriter != nil {
+		return harWriter.OutChan
+	}
+
+	return nil
+}
+
+func startPassiveTapper(harWriter *HarWriter) {
 	defer util.Run()()
 	if *debug {
 		outputLevel = 2
@@ -311,14 +327,11 @@ func StartPassiveTapper() {
 		}
 	}
 
-	var harWriter *HarWriter
 	if *dumpToHar {
-		harWriter = NewHarWriter(*HarOutputDir, *harEntriesPerFile)
 		harWriter.Start()
 		defer harWriter.Stop()
 	}
 
-
 	var dec gopacket.Decoder
 	var ok bool
 	decoder_name := *decoder

api/pkg/tap/tap_output.go

@@ -3,7 +3,6 @@ package tap
 import (
 	"bytes"
 	"encoding/json"
-	"flag"
 	"fmt"
 	"log"
 	"net/http"
@@ -200,7 +199,6 @@ func serveWs(hub *Hub, w http.ResponseWriter, r *http.Request) {
 }
 
 func startOutputServer(port string, messageCallback func([]byte)) {
-	flag.Parse()
 	hub = newHub(messageCallback)
 	go hub.run()
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {

api/start.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-./mizuagent -i any -hardump -hardir /tmp/mizuhars -harentriesperfile 5 -targets ${TAPPED_ADDRESSES}
+./mizuagent -i any -hardump -targets ${TAPPED_ADDRESSES}