diff --git a/agent/main.go b/agent/main.go index 063b150c1..84baf3b0f 100644 --- a/agent/main.go +++ b/agent/main.go @@ -4,6 +4,12 @@ import ( "encoding/json" "flag" "fmt" + "github.com/gin-contrib/static" + "github.com/gin-gonic/gin" + "github.com/gorilla/websocket" + "github.com/romana/rlog" + "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/tap" "mizuserver/pkg/api" "mizuserver/pkg/models" "mizuserver/pkg/routes" @@ -13,17 +19,9 @@ import ( "os" "os/signal" "strings" - - "github.com/gin-contrib/static" - "github.com/gin-gonic/gin" - "github.com/gorilla/websocket" - "github.com/romana/rlog" - "github.com/up9inc/mizu/shared" - "github.com/up9inc/mizu/tap" ) var shouldTap = flag.Bool("tap", false, "Run in tapper mode without API") -var demo = flag.Bool("demo", false, "Run in Demo mode with API") var apiServer = flag.Bool("api-server", false, "Run in API server mode with API") var standalone = flag.Bool("standalone", false, "Run in standalone tapper and API mode") var apiServerAddress = flag.String("api-server-address", "", "Address of mizu API server") @@ -36,12 +34,13 @@ func main() { if !*shouldTap && !*apiServer && !*standalone { panic("One of the flags --tap, --api or --standalone must be provided") } + if *standalone { harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts) filteredHarChannel := make(chan *tap.OutputChannelItem) go filterHarItems(harOutputChannel, filteredHarChannel, getTrafficFilteringOptions()) - go api.StartReadingEntries(filteredHarChannel, nil, false) + go api.StartReadingEntries(filteredHarChannel, nil) go api.StartReadingOutbound(outboundLinkOutputChannel) hostApi(nil) @@ -70,12 +69,7 @@ func main() { filteredHarChannel := make(chan *tap.OutputChannelItem) go filterHarItems(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions()) - if *demo { - workdir := "./hars" - go api.StartReadingEntries(filteredHarChannel, &workdir, true) - } else { - go api.StartReadingEntries(filteredHarChannel, nil, false) - } + go api.StartReadingEntries(filteredHarChannel, nil) hostApi(socketHarOutChannel) } diff --git a/agent/pkg/api/main.go b/agent/pkg/api/main.go index 94ca6f150..711b64bf2 100644 --- a/agent/pkg/api/main.go +++ b/agent/pkg/api/main.go @@ -5,7 +5,10 @@ import ( "context" "encoding/json" "fmt" - "math/rand" + "github.com/google/martian/har" + "github.com/romana/rlog" + "github.com/up9inc/mizu/tap" + "go.mongodb.org/mongo-driver/bson/primitive" "mizuserver/pkg/holder" "net/url" "os" @@ -14,11 +17,6 @@ import ( "strings" "time" - "github.com/google/martian/har" - "github.com/romana/rlog" - "github.com/up9inc/mizu/tap" - "go.mongodb.org/mongo-driver/bson/primitive" - "mizuserver/pkg/database" "mizuserver/pkg/models" "mizuserver/pkg/resolver" @@ -49,15 +47,15 @@ func init() { holder.SetResolver(res) } -func StartReadingEntries(harChannel <-chan *tap.OutputChannelItem, workingDir *string, demo bool) { +func StartReadingEntries(harChannel <-chan *tap.OutputChannelItem, workingDir *string) { if workingDir != nil && *workingDir != "" { - startReadingFiles(*workingDir, demo) + startReadingFiles(*workingDir) } else { startReadingChannel(harChannel) } } -func startReadingFiles(workingDir string, infiniteLoad bool) { +func startReadingFiles(workingDir string) { err := os.MkdirAll(workingDir, os.ModePerm) utils.CheckErr(err) @@ -88,23 +86,18 @@ func startReadingFiles(workingDir string, infiniteLoad bool) { utils.CheckErr(decErr) for _, entry := range inputHar.Log.Entries { - if infiniteLoad { - entry.StartedDateTime = time.Now().Add(20 * time.Millisecond) - } - time.Sleep(time.Millisecond * time.Duration(rand.Intn(300))) + time.Sleep(time.Millisecond * 250) connectionInfo := &tap.ConnectionInfo{ - ClientIP: fileInfo.Name(), + ClientIP: fileInfo.Name(), ClientPort: "", - ServerIP: "", + ServerIP: "", ServerPort: "", IsOutgoing: false, } saveHarToDb(entry, connectionInfo) } - if !infiniteLoad { - rmErr := os.Remove(inputFilePath) - utils.CheckErr(rmErr) - } + rmErr := os.Remove(inputFilePath) + utils.CheckErr(rmErr) } } @@ -125,6 +118,7 @@ func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) { } } + func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) { entryBytes, _ := json.Marshal(entry) serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL) @@ -202,5 +196,6 @@ func getEstimatedEntrySizeBytes(mizuEntry models.MizuEntry) int { sizeBytes += 8 // SizeBytes bytes sizeBytes += 1 // IsOutgoing bytes + return sizeBytes } diff --git a/cli/cmd/demo.go b/cli/cmd/demo.go deleted file mode 100644 index 6bd4493e2..000000000 --- a/cli/cmd/demo.go +++ /dev/null @@ -1,32 +0,0 @@ -package cmd - -import ( - "github.com/spf13/cobra" -) - -type MizuDemoOptions struct { - GuiPort uint16 - Analyze bool - AnalyzeDestination string -} - -var mizuDemoOptions = &MizuDemoOptions{} - -var demoCmd = &cobra.Command{ - Use: "demo", - Short: "Record ingoing traffic of a kubernetes pod", - Long: `Record the ingoing traffic of a kubernetes pod. -Supported protocols are HTTP and gRPC.`, - RunE: func(cmd *cobra.Command, args []string) error { - RunMizuTapDemo(mizuDemoOptions) - return nil - }, -} - -func init() { - rootCmd.AddCommand(demoCmd) - - demoCmd.Flags().Uint16VarP(&mizuDemoOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver") - demoCmd.Flags().BoolVar(&mizuDemoOptions.Analyze, "analyze", false, "Uploads traffic to UP9 cloud for further analysis (Beta)") - demoCmd.Flags().StringVar(&mizuDemoOptions.AnalyzeDestination, "dest", "up9.app", "Destination environment") -} diff --git a/cli/cmd/demoRunner.go b/cli/cmd/demoRunner.go deleted file mode 100644 index 59a28d68f..000000000 --- a/cli/cmd/demoRunner.go +++ /dev/null @@ -1,185 +0,0 @@ -package cmd - -import ( - "archive/zip" - "bytes" - "context" - "fmt" - "io" - "log" - "net/http" - "os" - "os/exec" - "os/signal" - "path/filepath" - "runtime" - "strings" - "syscall" - "time" - - "github.com/up9inc/mizu/cli/uiUtils" -) - -func RunMizuTapDemo(demoOptions *MizuDemoOptions) { - dir, _ := os.Getwd() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - downloadMizuDemo(dir) - - go callMizuDemo(ctx, cancel, dir, demoOptions) - if demoOptions.Analyze { - go analyze(demoOptions) - fmt.Printf(uiUtils.Purple, "mizu tap \"catalogue-.*|carts-[0-9].*|payment.*|shipping.*|user-[0-9].*\" -n sock-shop --analyze\n") - } else { - fmt.Printf(uiUtils.Purple, "mizu tap \"catalogue-.*|carts-[0-9].*|payment.*|shipping.*|user-[0-9].*\" -n sock-shop\n") - } - fmt.Println("Mizu will be available on http://localhost:8899 in a few seconds") - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - - select { - case <-ctx.Done(): - break - case <-sigChan: - cleanUpDemoResources(dir) - cancel() - } -} - -func cleanUpDemoResources(dir string) { - removeFile(fmt.Sprintf("%s/site.zip", dir)) - removeFile(fmt.Sprintf("%s/site", dir)) - removeFile(fmt.Sprintf("%s/apiserver.zip", dir)) - removeFile(fmt.Sprintf("%s/apiserver", dir)) - removeFile(fmt.Sprintf("%s/entries.db", dir)) - removeFile(fmt.Sprintf("%s/hars", dir)) - removeFile(fmt.Sprintf("%s/hars.zip", dir)) -} - -func removeFile(file string) { - err := os.RemoveAll(file) - if err != nil { - log.Fatal(err) - } -} - -func downloadMizuDemo(dir string) { - if runtime.GOOS != "darwin" && runtime.GOOS != "linux" { - panic("Platform not supported") - } - mizuApiURL := fmt.Sprintf("https://storage.googleapis.com/up9-mizu-demo-mode/apiserver-%s.zip", runtime.GOOS) - siteFileURL := "https://storage.googleapis.com/up9-mizu-demo-mode/site.zip" - harsURL := "https://storage.googleapis.com/up9-mizu-demo-mode/hars.zip" - - dirApi := fmt.Sprintf("%s/apiserver.zip", dir) - dirSite := fmt.Sprintf("%s/site.zip", dir) - dirHars := fmt.Sprintf("%s/hars.zip", dir) - - DownloadFile(dirApi, mizuApiURL) - DownloadFile(dirSite, siteFileURL) - DownloadFile(dirHars, harsURL) - - UnzipSite(dirSite, fmt.Sprintf("%s/", dir)) - UnzipSite(dirApi, fmt.Sprintf("%s/", dir)) - UnzipSite(dirHars, fmt.Sprintf("%s/", dir)) - allowExecutable(fmt.Sprintf("%s/apiserver", dir)) -} - -func DownloadFile(filepath string, url string) error { - - resp, err := http.Get(url) - if err != nil { - return err - } - defer resp.Body.Close() - out, err := os.Create(filepath) - if err != nil { - return err - } - defer out.Close() - _, err = io.Copy(out, resp.Body) - return err -} - -func UnzipSite(src string, dest string) ([]string, error) { - var filenames []string - - r, err := zip.OpenReader(src) - if err != nil { - return filenames, err - } - defer r.Close() - - for _, f := range r.File { - - fpath := filepath.Join(dest, f.Name) - if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) { - return filenames, fmt.Errorf("%s: illegal file path", fpath) - } - - filenames = append(filenames, fpath) - - if f.FileInfo().IsDir() { - os.MkdirAll(fpath, os.ModePerm) - continue - } - if err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil { - return filenames, err - } - - outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) - if err != nil { - return filenames, err - } - - rc, err := f.Open() - if err != nil { - return filenames, err - } - - _, err = io.Copy(outFile, rc) - outFile.Close() - rc.Close() - - if err != nil { - return filenames, err - } - } - return filenames, nil -} - -func allowExecutable(dir string) { - if err := os.Chmod(dir, 0755); err != nil { - log.Fatalln(err) - } -} - -func callMizuDemo(ctx context.Context, cancel context.CancelFunc, dir string, demoOptions *MizuDemoOptions) { - cmd := exec.Command(fmt.Sprintf("%s/apiserver", dir), "--aggregator", "--demo") - var out bytes.Buffer - - // set the output to our variable - cmd.Stdout = &out - err := cmd.Run() - if err != nil { - log.Println(err) - } -} - -func analyze(demoOptions *MizuDemoOptions) { - mizuProxiedUrl := getMizuCollectorProxiedHostAndPath(demoOptions.GuiPort) - for { - response, err := http.Get(fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=10", mizuProxiedUrl, demoOptions.AnalyzeDestination)) - if err != nil || response.StatusCode != 200 { - fmt.Printf(uiUtils.Red, "Mizu Not running, waiting 10 seconds before trying again\n") - } else { - fmt.Printf(uiUtils.Purple, "Traffic is uploading to UP9 cloud for further analsys\n") - break - } - time.Sleep(10 * time.Second) - } -} - -func getMizuCollectorProxiedHostAndPath(mizuPort uint16) string { - return fmt.Sprintf("localhost:%d", mizuPort) -}