diff --git a/.gitignore b/.gitignore index 5c657649f..ef835de11 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ build # Build directories build + +# Mac OS +.DS_Store diff --git a/Makefile b/Makefile index 171308c5b..9a6dce569 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ tap: ## build tap binary docker: ## build Docker image @(echo "building docker image" ) - docker build -t ${DOCKER_IMG}:${DOCKER_TAG} api + docker build -t ${DOCKER_IMG}:${DOCKER_TAG} . docker images ${DOCKER_IMG} publish: ## build and publish Mizu docker image & CLI diff --git a/api/main.go b/api/main.go index 24cdbbd05..65153a537 100644 --- a/api/main.go +++ b/api/main.go @@ -1,6 +1,8 @@ package main import ( + "flag" + "github.com/gofiber/fiber/v2" "mizuserver/pkg/inserter" "mizuserver/pkg/middleware" @@ -10,12 +12,14 @@ import ( ) func main() { + flag.Parse() - go tap.StartPassiveTapper() + harOutputChannel := tap.StartPassiveTapper() app := fiber.New() - go inserter.StartReadingFiles(*tap.HarOutputDir) // process to read files and insert to DB + // process to read files / channel and insert to DB + go inserter.StartReadingFiles(harOutputChannel, tap.HarOutputDir) middleware.FiberMiddleware(app) // Register Fiber's middleware for app. diff --git a/api/pkg/inserter/main.go b/api/pkg/inserter/main.go index 02601cba6..70c6e6652 100644 --- a/api/pkg/inserter/main.go +++ b/api/pkg/inserter/main.go @@ -17,7 +17,15 @@ import ( "time" ) -func StartReadingFiles(workingDir string) { +func StartReadingFiles(harChannel chan *har.Entry, workingDir *string) { + if workingDir != nil && *workingDir != "" { + startReadingFiles(*workingDir) + } else { + startReadingSChan(harChannel) + } +} + +func startReadingFiles(workingDir string) { err := os.MkdirAll(workingDir, os.ModePerm) utils.CheckErr(err) @@ -49,6 +57,12 @@ func StartReadingFiles(workingDir string) { } } +func startReadingSChan(harChannel chan *har.Entry) { + for entry := range harChannel { + SaveHarToDb(*entry, "") + } +} + func SaveHarToDb(entry har.Entry, source string) { entryBytes, _ := json.Marshal(entry) serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL) diff --git a/api/pkg/tap/har_writer.go b/api/pkg/tap/har_writer.go index 69c3c3b60..4620fcc82 100644 --- a/api/pkg/tap/har_writer.go +++ b/api/pkg/tap/har_writer.go @@ -2,6 +2,7 @@ package tap import ( "encoding/json" + "errors" "fmt" "net/http" "os" @@ -38,15 +39,15 @@ type HarFile struct { entryCount int } -func (f *HarFile) WriteEntry(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time) { +func NewEntry(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time) (*har.Entry, error) { // TODO: quick fix until TRA-3212 is implemented if request.URL == nil || request.Method == "" { - return + return nil, errors.New("Invalid request") } harRequest, err := har.NewRequest(request, true) if err != nil { SilentError("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)\n", err, err, err) - return + return nil, errors.New("Failed converting request to HAR") } // Martian copies http.Request.URL.String() to har.Request.URL. @@ -56,7 +57,7 @@ func (f *HarFile) WriteEntry(request *http.Request, requestTime time.Time, respo harResponse, err := har.NewResponse(response, true) if err != nil { SilentError("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)\n", err, err, err) - return + return nil, errors.New("Failed converting response to HAR") } totalTime := responseTime.Sub(requestTime).Round(time.Millisecond).Milliseconds() @@ -77,6 +78,10 @@ func (f *HarFile) WriteEntry(request *http.Request, requestTime time.Time, respo }, } + return &harEntry, nil +} + +func (f *HarFile) WriteEntry(harEntry *har.Entry) { harEntryJson, err := json.Marshal(harEntry) if err != nil { SilentError("har-entry-marshal", "Failed converting har entry object to JSON%s (%v,%+v)\n", err, err, err) @@ -131,6 +136,7 @@ func NewHarWriter(outputDir string, maxEntries int) *HarWriter { OutputDirPath: outputDir, MaxEntries: maxEntries, PairChan: make(chan *PairChanItem), + OutChan: make(chan *har.Entry, 1000), currentFile: nil, done: make(chan bool), } @@ -140,6 +146,7 @@ type HarWriter struct { OutputDirPath string MaxEntries int PairChan chan *PairChanItem + OutChan chan *har.Entry currentFile *HarFile done chan bool } @@ -154,20 +161,31 @@ func (hw *HarWriter) WritePair(request *http.Request, requestTime time.Time, res } func (hw *HarWriter) Start() { - if err := os.MkdirAll(hw.OutputDirPath, os.ModePerm); err != nil { - panic(fmt.Sprintf("Failed to create output directory: %s (%v,%+v)", err, err, err)) + if hw.OutputDirPath != "" { + if err := os.MkdirAll(hw.OutputDirPath, os.ModePerm); err != nil { + panic(fmt.Sprintf("Failed to create output directory: %s (%v,%+v)", err, err, err)) + } } go func() { for pair := range hw.PairChan { - if hw.currentFile == nil { - hw.openNewFile() + harEntry, err := NewEntry(pair.Request, pair.RequestTime, pair.Response, pair.ResponseTime) + if err != nil { + continue } - hw.currentFile.WriteEntry(pair.Request, pair.RequestTime, pair.Response, pair.ResponseTime) + if hw.OutputDirPath != "" { + if hw.currentFile == nil { + hw.openNewFile() + } - if hw.currentFile.GetEntryCount() >= hw.MaxEntries { - hw.closeFile() + hw.currentFile.WriteEntry(harEntry) + + if hw.currentFile.GetEntryCount() >= hw.MaxEntries { + hw.closeFile() + } + } else { + hw.OutChan <- harEntry } } diff --git a/api/pkg/tap/passive_tapper.go b/api/pkg/tap/passive_tapper.go index 6127277c5..ff0dd8177 100644 --- a/api/pkg/tap/passive_tapper.go +++ b/api/pkg/tap/passive_tapper.go @@ -29,6 +29,7 @@ import ( "github.com/google/gopacket/layers" // pulls in all layers decoders "github.com/google/gopacket/pcap" "github.com/google/gopacket/reassembly" + "github.com/google/martian/har" ) const AppPortsEnvVar = "APP_PORTS" @@ -96,7 +97,7 @@ var memprofile = flag.String("memprofile", "", "Write memory profile") // output var dumpToHar = flag.Bool("hardump", false, "Dump traffic to har files") -var HarOutputDir = flag.String("hardir", "output", "Directory in which to store output har files") +var HarOutputDir = flag.String("hardir", "", "Directory in which to store output har files") var harEntriesPerFile = flag.Int("harentriesperfile", 200, "Number of max number of har entries to store in each file") var reqResMatcher = createResponseRequestMatcher() // global @@ -198,7 +199,22 @@ func (c *Context) GetCaptureInfo() gopacket.CaptureInfo { return c.CaptureInfo } -func StartPassiveTapper() { +func StartPassiveTapper() chan *har.Entry { + var harWriter *HarWriter + if *dumpToHar { + harWriter = NewHarWriter(*HarOutputDir, *harEntriesPerFile) + } + + go startPassiveTapper(harWriter) + + if harWriter != nil { + return harWriter.OutChan + } + + return nil +} + +func startPassiveTapper(harWriter *HarWriter) { defer util.Run()() if *debug { outputLevel = 2 @@ -311,14 +327,11 @@ func StartPassiveTapper() { } } - var harWriter *HarWriter if *dumpToHar { - harWriter = NewHarWriter(*HarOutputDir, *harEntriesPerFile) harWriter.Start() defer harWriter.Stop() } - var dec gopacket.Decoder var ok bool decoder_name := *decoder diff --git a/api/pkg/tap/tap_output.go b/api/pkg/tap/tap_output.go index efbe874c6..561c445c0 100644 --- a/api/pkg/tap/tap_output.go +++ b/api/pkg/tap/tap_output.go @@ -3,7 +3,6 @@ package tap import ( "bytes" "encoding/json" - "flag" "fmt" "log" "net/http" @@ -200,7 +199,6 @@ func serveWs(hub *Hub, w http.ResponseWriter, r *http.Request) { } func startOutputServer(port string, messageCallback func([]byte)) { - flag.Parse() hub = newHub(messageCallback) go hub.run() http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { diff --git a/api/start.sh b/api/start.sh index f231f23c2..4b04b6d47 100755 --- a/api/start.sh +++ b/api/start.sh @@ -1,2 +1,2 @@ #!/bin/bash -./mizuagent -i any -hardump -hardir /tmp/mizuhars -harentriesperfile 5 -targets ${TAPPED_ADDRESSES} \ No newline at end of file +./mizuagent -i any -hardump -targets ${TAPPED_ADDRESSES}