diff --git a/agent/main.go b/agent/main.go index 84baf3b0f..063b150c1 100644 --- a/agent/main.go +++ b/agent/main.go @@ -4,12 +4,6 @@ import ( "encoding/json" "flag" "fmt" - "github.com/gin-contrib/static" - "github.com/gin-gonic/gin" - "github.com/gorilla/websocket" - "github.com/romana/rlog" - "github.com/up9inc/mizu/shared" - "github.com/up9inc/mizu/tap" "mizuserver/pkg/api" "mizuserver/pkg/models" "mizuserver/pkg/routes" @@ -19,9 +13,17 @@ import ( "os" "os/signal" "strings" + + "github.com/gin-contrib/static" + "github.com/gin-gonic/gin" + "github.com/gorilla/websocket" + "github.com/romana/rlog" + "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/tap" ) var shouldTap = flag.Bool("tap", false, "Run in tapper mode without API") +var demo = flag.Bool("demo", false, "Run in Demo mode with API") var apiServer = flag.Bool("api-server", false, "Run in API server mode with API") var standalone = flag.Bool("standalone", false, "Run in standalone tapper and API mode") var apiServerAddress = flag.String("api-server-address", "", "Address of mizu API server") @@ -34,13 +36,12 @@ func main() { if !*shouldTap && !*apiServer && !*standalone { panic("One of the flags --tap, --api or --standalone must be provided") } - if *standalone { harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts) filteredHarChannel := make(chan *tap.OutputChannelItem) go filterHarItems(harOutputChannel, filteredHarChannel, getTrafficFilteringOptions()) - go api.StartReadingEntries(filteredHarChannel, nil) + go api.StartReadingEntries(filteredHarChannel, nil, false) go api.StartReadingOutbound(outboundLinkOutputChannel) hostApi(nil) @@ -69,7 +70,12 @@ func main() { filteredHarChannel := make(chan *tap.OutputChannelItem) go filterHarItems(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions()) - go api.StartReadingEntries(filteredHarChannel, nil) + if *demo { + workdir := "./hars" + go api.StartReadingEntries(filteredHarChannel, &workdir, true) + } else { + go api.StartReadingEntries(filteredHarChannel, nil, false) + } hostApi(socketHarOutChannel) } diff --git a/agent/pkg/api/main.go b/agent/pkg/api/main.go index 711b64bf2..94ca6f150 100644 --- a/agent/pkg/api/main.go +++ b/agent/pkg/api/main.go @@ -5,10 +5,7 @@ import ( "context" "encoding/json" "fmt" - "github.com/google/martian/har" - "github.com/romana/rlog" - "github.com/up9inc/mizu/tap" - "go.mongodb.org/mongo-driver/bson/primitive" + "math/rand" "mizuserver/pkg/holder" "net/url" "os" @@ -17,6 +14,11 @@ import ( "strings" "time" + "github.com/google/martian/har" + "github.com/romana/rlog" + "github.com/up9inc/mizu/tap" + "go.mongodb.org/mongo-driver/bson/primitive" + "mizuserver/pkg/database" "mizuserver/pkg/models" "mizuserver/pkg/resolver" @@ -47,15 +49,15 @@ func init() { holder.SetResolver(res) } -func StartReadingEntries(harChannel <-chan *tap.OutputChannelItem, workingDir *string) { +func StartReadingEntries(harChannel <-chan *tap.OutputChannelItem, workingDir *string, demo bool) { if workingDir != nil && *workingDir != "" { - startReadingFiles(*workingDir) + startReadingFiles(*workingDir, demo) } else { startReadingChannel(harChannel) } } -func startReadingFiles(workingDir string) { +func startReadingFiles(workingDir string, infiniteLoad bool) { err := os.MkdirAll(workingDir, os.ModePerm) utils.CheckErr(err) @@ -86,18 +88,23 @@ func startReadingFiles(workingDir string) { utils.CheckErr(decErr) for _, entry := range inputHar.Log.Entries { - time.Sleep(time.Millisecond * 250) + if infiniteLoad { + entry.StartedDateTime = time.Now().Add(20 * time.Millisecond) + } + time.Sleep(time.Millisecond * time.Duration(rand.Intn(300))) connectionInfo := &tap.ConnectionInfo{ - ClientIP: fileInfo.Name(), + ClientIP: fileInfo.Name(), ClientPort: "", - ServerIP: "", + ServerIP: "", ServerPort: "", IsOutgoing: false, } saveHarToDb(entry, connectionInfo) } - rmErr := os.Remove(inputFilePath) - utils.CheckErr(rmErr) + if !infiniteLoad { + rmErr := os.Remove(inputFilePath) + utils.CheckErr(rmErr) + } } } @@ -118,7 +125,6 @@ func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) { } } - func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) { entryBytes, _ := json.Marshal(entry) serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL) @@ -196,6 +202,5 @@ func getEstimatedEntrySizeBytes(mizuEntry models.MizuEntry) int { sizeBytes += 8 // SizeBytes bytes sizeBytes += 1 // IsOutgoing bytes - return sizeBytes } diff --git a/cli/cmd/demo.go b/cli/cmd/demo.go new file mode 100644 index 000000000..6bd4493e2 --- /dev/null +++ b/cli/cmd/demo.go @@ -0,0 +1,32 @@ +package cmd + +import ( + "github.com/spf13/cobra" +) + +type MizuDemoOptions struct { + GuiPort uint16 + Analyze bool + AnalyzeDestination string +} + +var mizuDemoOptions = &MizuDemoOptions{} + +var demoCmd = &cobra.Command{ + Use: "demo", + Short: "Record ingoing traffic of a kubernetes pod", + Long: `Record the ingoing traffic of a kubernetes pod. +Supported protocols are HTTP and gRPC.`, + RunE: func(cmd *cobra.Command, args []string) error { + RunMizuTapDemo(mizuDemoOptions) + return nil + }, +} + +func init() { + rootCmd.AddCommand(demoCmd) + + demoCmd.Flags().Uint16VarP(&mizuDemoOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver") + demoCmd.Flags().BoolVar(&mizuDemoOptions.Analyze, "analyze", false, "Uploads traffic to UP9 cloud for further analysis (Beta)") + demoCmd.Flags().StringVar(&mizuDemoOptions.AnalyzeDestination, "dest", "up9.app", "Destination environment") +} diff --git a/cli/cmd/demoRunner.go b/cli/cmd/demoRunner.go new file mode 100644 index 000000000..59a28d68f --- /dev/null +++ b/cli/cmd/demoRunner.go @@ -0,0 +1,185 @@ +package cmd + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io" + "log" + "net/http" + "os" + "os/exec" + "os/signal" + "path/filepath" + "runtime" + "strings" + "syscall" + "time" + + "github.com/up9inc/mizu/cli/uiUtils" +) + +func RunMizuTapDemo(demoOptions *MizuDemoOptions) { + dir, _ := os.Getwd() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + downloadMizuDemo(dir) + + go callMizuDemo(ctx, cancel, dir, demoOptions) + if demoOptions.Analyze { + go analyze(demoOptions) + fmt.Printf(uiUtils.Purple, "mizu tap \"catalogue-.*|carts-[0-9].*|payment.*|shipping.*|user-[0-9].*\" -n sock-shop --analyze\n") + } else { + fmt.Printf(uiUtils.Purple, "mizu tap \"catalogue-.*|carts-[0-9].*|payment.*|shipping.*|user-[0-9].*\" -n sock-shop\n") + } + fmt.Println("Mizu will be available on http://localhost:8899 in a few seconds") + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + select { + case <-ctx.Done(): + break + case <-sigChan: + cleanUpDemoResources(dir) + cancel() + } +} + +func cleanUpDemoResources(dir string) { + removeFile(fmt.Sprintf("%s/site.zip", dir)) + removeFile(fmt.Sprintf("%s/site", dir)) + removeFile(fmt.Sprintf("%s/apiserver.zip", dir)) + removeFile(fmt.Sprintf("%s/apiserver", dir)) + removeFile(fmt.Sprintf("%s/entries.db", dir)) + removeFile(fmt.Sprintf("%s/hars", dir)) + removeFile(fmt.Sprintf("%s/hars.zip", dir)) +} + +func removeFile(file string) { + err := os.RemoveAll(file) + if err != nil { + log.Fatal(err) + } +} + +func downloadMizuDemo(dir string) { + if runtime.GOOS != "darwin" && runtime.GOOS != "linux" { + panic("Platform not supported") + } + mizuApiURL := fmt.Sprintf("https://storage.googleapis.com/up9-mizu-demo-mode/apiserver-%s.zip", runtime.GOOS) + siteFileURL := "https://storage.googleapis.com/up9-mizu-demo-mode/site.zip" + harsURL := "https://storage.googleapis.com/up9-mizu-demo-mode/hars.zip" + + dirApi := fmt.Sprintf("%s/apiserver.zip", dir) + dirSite := fmt.Sprintf("%s/site.zip", dir) + dirHars := fmt.Sprintf("%s/hars.zip", dir) + + DownloadFile(dirApi, mizuApiURL) + DownloadFile(dirSite, siteFileURL) + DownloadFile(dirHars, harsURL) + + UnzipSite(dirSite, fmt.Sprintf("%s/", dir)) + UnzipSite(dirApi, fmt.Sprintf("%s/", dir)) + UnzipSite(dirHars, fmt.Sprintf("%s/", dir)) + allowExecutable(fmt.Sprintf("%s/apiserver", dir)) +} + +func DownloadFile(filepath string, url string) error { + + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + out, err := os.Create(filepath) + if err != nil { + return err + } + defer out.Close() + _, err = io.Copy(out, resp.Body) + return err +} + +func UnzipSite(src string, dest string) ([]string, error) { + var filenames []string + + r, err := zip.OpenReader(src) + if err != nil { + return filenames, err + } + defer r.Close() + + for _, f := range r.File { + + fpath := filepath.Join(dest, f.Name) + if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) { + return filenames, fmt.Errorf("%s: illegal file path", fpath) + } + + filenames = append(filenames, fpath) + + if f.FileInfo().IsDir() { + os.MkdirAll(fpath, os.ModePerm) + continue + } + if err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil { + return filenames, err + } + + outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return filenames, err + } + + rc, err := f.Open() + if err != nil { + return filenames, err + } + + _, err = io.Copy(outFile, rc) + outFile.Close() + rc.Close() + + if err != nil { + return filenames, err + } + } + return filenames, nil +} + +func allowExecutable(dir string) { + if err := os.Chmod(dir, 0755); err != nil { + log.Fatalln(err) + } +} + +func callMizuDemo(ctx context.Context, cancel context.CancelFunc, dir string, demoOptions *MizuDemoOptions) { + cmd := exec.Command(fmt.Sprintf("%s/apiserver", dir), "--aggregator", "--demo") + var out bytes.Buffer + + // set the output to our variable + cmd.Stdout = &out + err := cmd.Run() + if err != nil { + log.Println(err) + } +} + +func analyze(demoOptions *MizuDemoOptions) { + mizuProxiedUrl := getMizuCollectorProxiedHostAndPath(demoOptions.GuiPort) + for { + response, err := http.Get(fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=10", mizuProxiedUrl, demoOptions.AnalyzeDestination)) + if err != nil || response.StatusCode != 200 { + fmt.Printf(uiUtils.Red, "Mizu Not running, waiting 10 seconds before trying again\n") + } else { + fmt.Printf(uiUtils.Purple, "Traffic is uploading to UP9 cloud for further analsys\n") + break + } + time.Sleep(10 * time.Second) + } +} + +func getMizuCollectorProxiedHostAndPath(mizuPort uint16) string { + return fmt.Sprintf("localhost:%d", mizuPort) +}