From b84c698c1a4011118f45c6d6a1121d9746474326 Mon Sep 17 00:00:00 2001 From: Igor Gov Date: Tue, 29 Jun 2021 17:05:44 +0300 Subject: [PATCH 01/12] Mizu tap analyze --- api/go.mod | 5 +- api/go.sum | 1 - api/pkg/controllers/entries_controller.go | 77 ++++++++++++++++++++++- api/pkg/models/models.go | 5 ++ api/pkg/routes/public_routes.go | 2 +- cli/cmd/tap.go | 5 +- cli/cmd/tapRunner.go | 40 ++++++++++++ 7 files changed, 125 insertions(+), 10 deletions(-) diff --git a/api/go.mod b/api/go.mod index f75ea5920..06b66bf96 100644 --- a/api/go.mod +++ b/api/go.mod @@ -11,16 +11,12 @@ require ( github.com/go-playground/universal-translator v0.17.0 github.com/go-playground/validator/v10 v10.5.0 github.com/gofiber/fiber/v2 v2.8.0 - github.com/google/gopacket v1.1.19 github.com/google/martian v2.1.0+incompatible github.com/gorilla/websocket v1.4.2 github.com/leodido/go-urn v1.2.1 // indirect - github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 - github.com/patrickmn/go-cache v2.1.0+incompatible github.com/up9inc/mizu/shared v0.0.0 github.com/up9inc/mizu/tap v0.0.0 go.mongodb.org/mongo-driver v1.5.1 - golang.org/x/net v0.0.0-20210421230115-4e50805a0758 gorm.io/driver/sqlite v1.1.4 gorm.io/gorm v1.21.8 k8s.io/api v0.21.0 @@ -29,4 +25,5 @@ require ( ) replace github.com/up9inc/mizu/shared v0.0.0 => ../shared + replace github.com/up9inc/mizu/tap v0.0.0 => ../tap diff --git a/api/go.sum b/api/go.sum index efa2d4285..14a3ed94e 100644 --- a/api/go.sum +++ b/api/go.sum @@ -251,7 +251,6 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU= github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= -github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index 282271b65..86359f55d 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -1,14 +1,20 @@ package controllers import ( + "bytes" + "compress/zlib" "encoding/json" "fmt" "github.com/gofiber/fiber/v2" "github.com/google/martian/har" + "io/ioutil" + "log" "mizuserver/pkg/database" "mizuserver/pkg/models" "mizuserver/pkg/utils" "mizuserver/pkg/validation" + "net/http" + "net/url" "time" ) @@ -138,9 +144,70 @@ func GetHARs(c *fiber.Ctx) error { return c.Status(fiber.StatusOK).SendStream(buffer) } +func uploadEntriesImpl(token string, model string) { + baseUrl := "igorgov-dev.dev.testr.io" + sleepTime := int64(time.Second * 10) + + var timestampFrom int64 = 0 + + for { + timestampTo := time.Now().UnixNano() / int64(time.Millisecond) + fmt.Printf("Getting entries from %v, to %v\n", timestampFrom, timestampTo) + entriesArray := getEntriesFromDb(timestampFrom, timestampTo) + + if len(entriesArray) > 0 { + fmt.Printf("About to upload %v entries\n", len(entriesArray)) + + body, jMarshalErr := json.Marshal(entriesArray) + if jMarshalErr != nil { + log.Fatal(jMarshalErr) + } + + var in bytes.Buffer + w := zlib.NewWriter(&in) + w.Write(body) + w.Close() + reqBody := ioutil.NopCloser(bytes.NewReader(in.Bytes())) + + postUrl, _ := url.Parse("https://traffic." + baseUrl + "/dumpTrafficBulk/" + model) + req := &http.Request{ + Method: "POST", + URL: postUrl, + Header: map[string][]string{ + "Content-Encoding": {"deflate"}, + "Content-Type": {"application/octet-stream"}, + "Guest-Auth": {token}, + }, + Body: reqBody, + } + _, postErr := http.DefaultClient.Do(req) + if postErr != nil { + log.Fatal(postErr) + } + } else { + fmt.Println("Nothing to upload") + } + + fmt.Println("Sleeping for " + string(sleepTime) + "seconds...") + time.Sleep(time.Duration(sleepTime)) + timestampFrom = timestampTo + } +} + +func UploadEntries(c *fiber.Ctx) error { + entriesFilter := &models.UploadEntriesRequestBody{} + if err := c.QueryParser(entriesFilter); err != nil { + return c.Status(fiber.StatusBadRequest).JSON(err) + } + if err := validation.Validate(entriesFilter); err != nil { + return c.Status(fiber.StatusBadRequest).JSON(err) + } + go uploadEntriesImpl(entriesFilter.Token, entriesFilter.Model) + return c.Status(fiber.StatusOK).SendString("OK") +} + func GetFullEntries(c *fiber.Ctx) error { entriesFilter := &models.HarFetchRequestBody{} - order := OrderDesc if err := c.QueryParser(entriesFilter); err != nil { return c.Status(fiber.StatusBadRequest).JSON(err) } @@ -162,6 +229,12 @@ func GetFullEntries(c *fiber.Ctx) error { timestampTo = entriesFilter.To } + entriesArray := getEntriesFromDb(timestampFrom, timestampTo) + return c.Status(fiber.StatusOK).JSON(entriesArray) +} + +func getEntriesFromDb(timestampFrom int64, timestampTo int64) []har.Entry { + order := OrderDesc var entries []models.MizuEntry database.GetEntriesTable(). Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)). @@ -179,7 +252,7 @@ func GetFullEntries(c *fiber.Ctx) error { _ = json.Unmarshal([]byte(entryData.Entry), &harEntry) entriesArray = append(entriesArray, harEntry) } - return c.Status(fiber.StatusOK).JSON(entriesArray) + return entriesArray } func GetEntry(c *fiber.Ctx) error { diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index 108801e1e..cf588fcec 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -49,6 +49,11 @@ type EntriesFilter struct { Timestamp int64 `query:"timestamp" validate:"required,min=1"` } +type UploadEntriesRequestBody struct { + Token string `query:"token"` + Model string `query:"model"` +} + type HarFetchRequestBody struct { From int64 `query:"from"` To int64 `query:"to"` diff --git a/api/pkg/routes/public_routes.go b/api/pkg/routes/public_routes.go index df589a62c..ae78d96db 100644 --- a/api/pkg/routes/public_routes.go +++ b/api/pkg/routes/public_routes.go @@ -12,7 +12,7 @@ func EntriesRoutes(fiberApp *fiber.App) { routeGroup.Get("/entries", controllers.GetEntries) // get entries (base/thin entries) routeGroup.Get("/entries/:entryId", controllers.GetEntry) // get single (full) entry routeGroup.Get("/exportEntries", controllers.GetFullEntries) - + routeGroup.Get("/uploadEntries", controllers.UploadEntries) routeGroup.Get("/har", controllers.GetHARs) diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index c3e6fb747..05f03072e 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -15,6 +15,7 @@ type MizuTapOptions struct { GuiPort uint16 Namespace string AllNamespaces bool + Analyze bool KubeConfigPath string MizuImage string MizuPodPort uint16 @@ -22,7 +23,6 @@ type MizuTapOptions struct { TapOutgoing bool } - var mizuTapOptions = &MizuTapOptions{} var direction string @@ -30,7 +30,7 @@ var tapCmd = &cobra.Command{ Use: "tap [POD REGEX]", Short: "Record ingoing traffic of a kubernetes pod", Long: `Record the ingoing traffic of a kubernetes pod. - Supported protocols are HTTP and gRPC.`, +Supported protocols are HTTP and gRPC.`, RunE: func(cmd *cobra.Command, args []string) error { if len(args) == 0 { return errors.New("POD REGEX argument is required") @@ -62,6 +62,7 @@ func init() { tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver") tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector") + tapCmd.Flags().BoolVarP(&mizuTapOptions.Analyze, "analyze", "", false, "Analyze traffic") tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces") tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file") tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector") diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 76011e1dc..cf9101f69 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -2,7 +2,10 @@ package cmd import ( "context" + "encoding/json" "fmt" + "io/ioutil" + "net/http" "os" "os/signal" "regexp" @@ -79,6 +82,32 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { waitForFinish(ctx, cancel) } +type guestToken struct { + Token string `json:"token"` + Model string `json:"model"` +} + +func CreateAnonymousToken(baseUrl string) guestToken { + resp, httpErr := http.Get("https://trcc." + baseUrl + "/anonymous/token") + if httpErr != nil { + fmt.Println(httpErr) + } + + body, ioErr := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if ioErr != nil { + fmt.Println(ioErr) + } + token := guestToken{} + jsonErr := json.Unmarshal(body, &token) + if jsonErr != nil { + fmt.Println(jsonErr) + } + fmt.Println("Token:", token.Token, "model:", token.Model) + + return token +} + func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error { if err := createMizuAggregator(ctx, kubernetesProvider, tappingOptions, mizuApiFilteringOptions); err != nil { return err @@ -244,6 +273,17 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi var err error portForward, err = kubernetes.NewPortForward(kubernetesProvider, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.GuiPort, tappingOptions.MizuPodPort, cancel) fmt.Printf("Web interface is now available at http://localhost:%d\n", tappingOptions.GuiPort) + + if tappingOptions.Analyze { + baseUrl := "igorgov-dev.dev.testr.io" + token := CreateAnonymousToken(baseUrl) + _, err := http.Get("http://localhost:8899/api/uploadEntries") + if err != nil { + fmt.Println(err) + } + fmt.Println("https://" + baseUrl + "/share/" + token.Token) + } + if err != nil { fmt.Printf("error forwarding port to pod %s\n", err) cancel() From 5f603e3291a7257a4fe21d81157ac426dc032374 Mon Sep 17 00:00:00 2001 From: Roee Gadot Date: Tue, 29 Jun 2021 19:16:46 +0300 Subject: [PATCH 02/12] improvements and fixes --- api/pkg/controllers/entries_controller.go | 21 ++++++++++++--------- api/pkg/models/models.go | 1 + cli/cmd/tap.go | 4 +++- cli/cmd/tapRunner.go | 9 ++++----- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index 86359f55d..507831b37 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -144,9 +144,8 @@ func GetHARs(c *fiber.Ctx) error { return c.Status(fiber.StatusOK).SendStream(buffer) } -func uploadEntriesImpl(token string, model string) { - baseUrl := "igorgov-dev.dev.testr.io" - sleepTime := int64(time.Second * 10) +func uploadEntriesImpl(token string, model string, envPrefix string) { + sleepTime := time.Second * 10 var timestampFrom int64 = 0 @@ -165,11 +164,12 @@ func uploadEntriesImpl(token string, model string) { var in bytes.Buffer w := zlib.NewWriter(&in) - w.Write(body) - w.Close() + _, _ = w.Write(body) + _ = w.Close() reqBody := ioutil.NopCloser(bytes.NewReader(in.Bytes())) - postUrl, _ := url.Parse("https://traffic." + baseUrl + "/dumpTrafficBulk/" + model) + postUrl, _ := url.Parse("https://traffic." + envPrefix + "/dumpTrafficBulk/" + model) + fmt.Println(postUrl) req := &http.Request{ Method: "POST", URL: postUrl, @@ -184,12 +184,15 @@ func uploadEntriesImpl(token string, model string) { if postErr != nil { log.Fatal(postErr) } + + fmt.Printf("Finish uploading %v entries to %s\n", len(entriesArray), postUrl) + } else { fmt.Println("Nothing to upload") } - fmt.Println("Sleeping for " + string(sleepTime) + "seconds...") - time.Sleep(time.Duration(sleepTime)) + fmt.Printf("Sleeping for %v...\n", sleepTime) + time.Sleep(sleepTime) timestampFrom = timestampTo } } @@ -202,7 +205,7 @@ func UploadEntries(c *fiber.Ctx) error { if err := validation.Validate(entriesFilter); err != nil { return c.Status(fiber.StatusBadRequest).JSON(err) } - go uploadEntriesImpl(entriesFilter.Token, entriesFilter.Model) + go uploadEntriesImpl(entriesFilter.Token, entriesFilter.Model, entriesFilter.Dest) return c.Status(fiber.StatusOK).SendString("OK") } diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index cf588fcec..b36ea7528 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -52,6 +52,7 @@ type EntriesFilter struct { type UploadEntriesRequestBody struct { Token string `query:"token"` Model string `query:"model"` + Dest string `query:"dest"` } type HarFetchRequestBody struct { diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 05f03072e..bd649baa9 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -16,6 +16,7 @@ type MizuTapOptions struct { Namespace string AllNamespaces bool Analyze bool + AnalyzeDestination string KubeConfigPath string MizuImage string MizuPodPort uint16 @@ -62,7 +63,8 @@ func init() { tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver") tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector") - tapCmd.Flags().BoolVarP(&mizuTapOptions.Analyze, "analyze", "", false, "Analyze traffic") + tapCmd.Flags().BoolVar(&mizuTapOptions.Analyze, "analyze", false, "Analyze traffic") + tapCmd.Flags().StringVar(&mizuTapOptions.AnalyzeDestination, "dest", "up9.app", "Destination environment") tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces") tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file") tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector") diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index cf9101f69..c57b29a01 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -74,7 +74,7 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { return } - go portForwardApiPod(ctx, kubernetesProvider, cancel, tappingOptions) // TODO convert this to job for built in pod ttl or have the running app handle this + go portForwardApiPod(ctx, kubernetesProvider, cancel, tappingOptions, ) // TODO convert this to job for built in pod ttl or have the running app handle this go watchPodsForTapping(ctx, kubernetesProvider, cancel, podRegexQuery, tappingOptions) go syncApiStatus(ctx, cancel, tappingOptions) @@ -275,13 +275,12 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi fmt.Printf("Web interface is now available at http://localhost:%d\n", tappingOptions.GuiPort) if tappingOptions.Analyze { - baseUrl := "igorgov-dev.dev.testr.io" - token := CreateAnonymousToken(baseUrl) - _, err := http.Get("http://localhost:8899/api/uploadEntries") + token := CreateAnonymousToken(tappingOptions.AnalyzeDestination) + _, err := http.Get(fmt.Sprintf("http://localhost:8899/api/uploadEntries?token=%s&model=%s&dest=%s", token.Token, token.Model, tappingOptions.AnalyzeDestination)) if err != nil { fmt.Println(err) } - fmt.Println("https://" + baseUrl + "/share/" + token.Token) + fmt.Println("https://" + tappingOptions.AnalyzeDestination + "/share/" + token.Token) } if err != nil { From fa733025dcac74107b85f4e5deec020cba6c7bc5 Mon Sep 17 00:00:00 2001 From: Roee Gadot Date: Wed, 30 Jun 2021 07:56:52 +0300 Subject: [PATCH 03/12] small changes --- api/pkg/controllers/entries_controller.go | 5 +- cli/cmd/tapRunner.go | 66 +++++++++++------------ 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index 507831b37..fa1306c7f 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -168,10 +168,10 @@ func uploadEntriesImpl(token string, model string, envPrefix string) { _ = w.Close() reqBody := ioutil.NopCloser(bytes.NewReader(in.Bytes())) - postUrl, _ := url.Parse("https://traffic." + envPrefix + "/dumpTrafficBulk/" + model) + postUrl, _ := url.Parse(fmt.Sprintf("https://traffic.%s/dumpTrafficBulk/%s", envPrefix, model)) fmt.Println(postUrl) req := &http.Request{ - Method: "POST", + Method: http.MethodPost, URL: postUrl, Header: map[string][]string{ "Content-Encoding": {"deflate"}, @@ -184,7 +184,6 @@ func uploadEntriesImpl(token string, model string, envPrefix string) { if postErr != nil { log.Fatal(postErr) } - fmt.Printf("Finish uploading %v entries to %s\n", len(entriesArray), postUrl) } else { diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index c57b29a01..b4d6cca87 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" "net/http" "os" "os/signal" @@ -82,30 +81,32 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { waitForFinish(ctx, cancel) } -type guestToken struct { +type GuestToken struct { Token string `json:"token"` Model string `json:"model"` } -func CreateAnonymousToken(baseUrl string) guestToken { - resp, httpErr := http.Get("https://trcc." + baseUrl + "/anonymous/token") - if httpErr != nil { - fmt.Println(httpErr) + +func getGuestToken(url string, target *GuestToken) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + return json.NewDecoder(resp.Body).Decode(target) +} + +func CreateAnonymousToken(envPrefix string) (*GuestToken, error) { + tokenUrl := fmt.Sprintf("https://trcc.%v/anonymous/token", envPrefix) + token := &GuestToken{} + if err := getGuestToken(tokenUrl, token); err != nil { + fmt.Println(err) + return nil, err } - body, ioErr := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if ioErr != nil { - fmt.Println(ioErr) - } - token := guestToken{} - jsonErr := json.Unmarshal(body, &token) - if jsonErr != nil { - fmt.Println(jsonErr) - } fmt.Println("Token:", token.Token, "model:", token.Model) - - return token + return token, nil } func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error { @@ -270,22 +271,21 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi case modifiedPod := <-modified: if modifiedPod.Status.Phase == "Running" && !isPodReady { isPodReady = true - var err error - portForward, err = kubernetes.NewPortForward(kubernetesProvider, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.GuiPort, tappingOptions.MizuPodPort, cancel) - fmt.Printf("Web interface is now available at http://localhost:%d\n", tappingOptions.GuiPort) - - if tappingOptions.Analyze { - token := CreateAnonymousToken(tappingOptions.AnalyzeDestination) - _, err := http.Get(fmt.Sprintf("http://localhost:8899/api/uploadEntries?token=%s&model=%s&dest=%s", token.Token, token.Model, tappingOptions.AnalyzeDestination)) - if err != nil { - fmt.Println(err) - } - fmt.Println("https://" + tappingOptions.AnalyzeDestination + "/share/" + token.Token) - } - - if err != nil { - fmt.Printf("error forwarding port to pod %s\n", err) + var portForwardCreateError error + if portForward, portForwardCreateError = kubernetes.NewPortForward(kubernetesProvider, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.GuiPort, tappingOptions.MizuPodPort, cancel); portForwardCreateError != nil { + fmt.Printf("error forwarding port to pod %s\n", portForwardCreateError) cancel() + } else { + fmt.Printf("Web interface is now available at http://localhost:%d\n", tappingOptions.GuiPort) + + if tappingOptions.Analyze { + token, _ := CreateAnonymousToken(tappingOptions.AnalyzeDestination) + if _, err := http.Get(fmt.Sprintf("http://localhost:%d/api/uploadEntries?token=%s&model=%s&dest=%s", tappingOptions.GuiPort, token.Token, token.Model, tappingOptions.AnalyzeDestination)); err != nil { + fmt.Println(err) + } else { + fmt.Println("https://" + tappingOptions.AnalyzeDestination + "/share/" + token.Token) + } + } } } From feb386ba1fb00a023330f65c0d5353528399abcb Mon Sep 17 00:00:00 2001 From: Igor Gov Date: Wed, 30 Jun 2021 12:01:02 +0300 Subject: [PATCH 04/12] Mizu tap analyze grooming --- cli/cmd/tap.go | 4 ++-- cli/cmd/tapRunner.go | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index bd649baa9..c23afd56f 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -16,7 +16,7 @@ type MizuTapOptions struct { Namespace string AllNamespaces bool Analyze bool - AnalyzeDestination string + AnalyzeDestination string KubeConfigPath string MizuImage string MizuPodPort uint16 @@ -63,7 +63,7 @@ func init() { tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver") tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector") - tapCmd.Flags().BoolVar(&mizuTapOptions.Analyze, "analyze", false, "Analyze traffic") + tapCmd.Flags().BoolVar(&mizuTapOptions.Analyze, "analyze", false, "Uploads traffic to UP9 cloud for further analysis") tapCmd.Flags().StringVar(&mizuTapOptions.AnalyzeDestination, "dest", "up9.app", "Destination environment") tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces") tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file") diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index b4d6cca87..7019c4715 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -73,7 +73,7 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { return } - go portForwardApiPod(ctx, kubernetesProvider, cancel, tappingOptions, ) // TODO convert this to job for built in pod ttl or have the running app handle this + go portForwardApiPod(ctx, kubernetesProvider, cancel, tappingOptions) // TODO convert this to job for built in pod ttl or have the running app handle this go watchPodsForTapping(ctx, kubernetesProvider, cancel, podRegexQuery, tappingOptions) go syncApiStatus(ctx, cancel, tappingOptions) @@ -86,7 +86,6 @@ type GuestToken struct { Model string `json:"model"` } - func getGuestToken(url string, target *GuestToken) error { resp, err := http.Get(url) if err != nil { @@ -104,8 +103,6 @@ func CreateAnonymousToken(envPrefix string) (*GuestToken, error) { fmt.Println(err) return nil, err } - - fmt.Println("Token:", token.Token, "model:", token.Model) return token, nil } @@ -283,6 +280,7 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi if _, err := http.Get(fmt.Sprintf("http://localhost:%d/api/uploadEntries?token=%s&model=%s&dest=%s", tappingOptions.GuiPort, token.Token, token.Model, tappingOptions.AnalyzeDestination)); err != nil { fmt.Println(err) } else { + fmt.Println("Staring to upload and analyze the data, it may take a few minutes") fmt.Println("https://" + tappingOptions.AnalyzeDestination + "/share/" + token.Token) } } From b762e3c19499a788c0db95a6fdeb3b16c8ccdabb Mon Sep 17 00:00:00 2001 From: nimrod-up9 <59927337+nimrod-up9@users.noreply.github.com> Date: Wed, 30 Jun 2021 12:18:29 +0300 Subject: [PATCH 05/12] New icons for direction (#88) * Replaced direction icons with designed icons. Match color to status code. * Took care of padding. Added seperator line. * Removed 1 div level and unnecessary properties in sass. * Removed ; * Changed to fixed svgs. --- ui/src/components/HarEntry.tsx | 38 ++++++++++++++----- ui/src/components/StatusCode.tsx | 14 +++++-- .../assets/ingoing-traffic-failure.svg | 5 +++ .../assets/ingoing-traffic-neutral.svg | 5 +++ .../assets/ingoing-traffic-success.svg | 5 +++ ui/src/components/assets/ingoing-traffic.svg | 1 - .../assets/outgoing-traffic-failure.svg | 5 +++ .../assets/outgoing-traffic-neutral.svg | 5 +++ .../assets/outgoing-traffic-success.svg | 5 +++ ui/src/components/assets/outgoing-traffic.svg | 1 - ui/src/components/style/HarEntry.module.sass | 18 +++------ 11 files changed, 75 insertions(+), 27 deletions(-) create mode 100644 ui/src/components/assets/ingoing-traffic-failure.svg create mode 100644 ui/src/components/assets/ingoing-traffic-neutral.svg create mode 100644 ui/src/components/assets/ingoing-traffic-success.svg delete mode 100644 ui/src/components/assets/ingoing-traffic.svg create mode 100644 ui/src/components/assets/outgoing-traffic-failure.svg create mode 100644 ui/src/components/assets/outgoing-traffic-neutral.svg create mode 100644 ui/src/components/assets/outgoing-traffic-success.svg delete mode 100644 ui/src/components/assets/outgoing-traffic.svg diff --git a/ui/src/components/HarEntry.tsx b/ui/src/components/HarEntry.tsx index 95f3c6ffd..368ac92cb 100644 --- a/ui/src/components/HarEntry.tsx +++ b/ui/src/components/HarEntry.tsx @@ -1,9 +1,13 @@ import React from "react"; import styles from './style/HarEntry.module.sass'; -import StatusCode from "./StatusCode"; +import StatusCode, {getClassification, StatusCodeClassification} from "./StatusCode"; import {EndpointPath} from "./EndpointPath"; -import ingoingIcon from "./assets/ingoing-traffic.svg" -import outgoingIcon from "./assets/outgoing-traffic.svg" +import ingoingIconSuccess from "./assets/ingoing-traffic-success.svg" +import ingoingIconFailure from "./assets/ingoing-traffic-failure.svg" +import ingoingIconNeutral from "./assets/ingoing-traffic-neutral.svg" +import outgoingIconSuccess from "./assets/outgoing-traffic-success.svg" +import outgoingIconFailure from "./assets/outgoing-traffic-failure.svg" +import outgoingIconNeutral from "./assets/outgoing-traffic-neutral.svg" interface HAREntry { method?: string, @@ -24,6 +28,26 @@ interface HAREntryProps { } export const HarEntry: React.FC = ({entry, setFocusedEntryId, isSelected}) => { + const classification = getClassification(entry.statusCode) + let ingoingIcon; + let outgoingIcon; + switch(classification) { + case StatusCodeClassification.SUCCESS: { + ingoingIcon = ingoingIconSuccess; + outgoingIcon = outgoingIconSuccess; + break; + } + case StatusCodeClassification.FAILURE: { + ingoingIcon = ingoingIconFailure; + outgoingIcon = outgoingIconFailure; + break; + } + case StatusCodeClassification.NEUTRAL: { + ingoingIcon = ingoingIconNeutral; + outgoingIcon = outgoingIconNeutral; + break; + } + } return <>
setFocusedEntryId(entry.id)}> @@ -38,13 +62,9 @@ export const HarEntry: React.FC = ({entry, setFocusedEntryId, isS
{entry.isOutgoing ? -
- outgoing traffic -
+ outgoing traffic : -
- ingoing traffic -
+ ingoing traffic }
{new Date(+entry.timestamp)?.toLocaleString()}
diff --git a/ui/src/components/StatusCode.tsx b/ui/src/components/StatusCode.tsx index 376a277ed..2230a9b8a 100644 --- a/ui/src/components/StatusCode.tsx +++ b/ui/src/components/StatusCode.tsx @@ -1,7 +1,7 @@ import React from "react"; import styles from './style/StatusCode.module.sass'; -enum StatusCodeClassification { +export enum StatusCodeClassification { SUCCESS = "success", FAILURE = "failure", NEUTRAL = "neutral" @@ -14,6 +14,12 @@ interface HAREntryProps { const StatusCode: React.FC = ({statusCode}) => { + const classification = getClassification(statusCode) + + return {statusCode} +}; + +export function getClassification(statusCode: number): string { let classification = StatusCodeClassification.NEUTRAL; if (statusCode >= 200 && statusCode <= 399) { @@ -22,7 +28,7 @@ const StatusCode: React.FC = ({statusCode}) => { classification = StatusCodeClassification.FAILURE; } - return {statusCode} -}; + return classification +} -export default StatusCode; \ No newline at end of file +export default StatusCode; diff --git a/ui/src/components/assets/ingoing-traffic-failure.svg b/ui/src/components/assets/ingoing-traffic-failure.svg new file mode 100644 index 000000000..3bb4e26f2 --- /dev/null +++ b/ui/src/components/assets/ingoing-traffic-failure.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/ui/src/components/assets/ingoing-traffic-neutral.svg b/ui/src/components/assets/ingoing-traffic-neutral.svg new file mode 100644 index 000000000..9720d65f2 --- /dev/null +++ b/ui/src/components/assets/ingoing-traffic-neutral.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/ui/src/components/assets/ingoing-traffic-success.svg b/ui/src/components/assets/ingoing-traffic-success.svg new file mode 100644 index 000000000..f1c745e64 --- /dev/null +++ b/ui/src/components/assets/ingoing-traffic-success.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/ui/src/components/assets/ingoing-traffic.svg b/ui/src/components/assets/ingoing-traffic.svg deleted file mode 100644 index 7aaded9a8..000000000 --- a/ui/src/components/assets/ingoing-traffic.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ui/src/components/assets/outgoing-traffic-failure.svg b/ui/src/components/assets/outgoing-traffic-failure.svg new file mode 100644 index 000000000..0a4f7cfb3 --- /dev/null +++ b/ui/src/components/assets/outgoing-traffic-failure.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/ui/src/components/assets/outgoing-traffic-neutral.svg b/ui/src/components/assets/outgoing-traffic-neutral.svg new file mode 100644 index 000000000..8f952e353 --- /dev/null +++ b/ui/src/components/assets/outgoing-traffic-neutral.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/ui/src/components/assets/outgoing-traffic-success.svg b/ui/src/components/assets/outgoing-traffic-success.svg new file mode 100644 index 000000000..477fb7350 --- /dev/null +++ b/ui/src/components/assets/outgoing-traffic-success.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/ui/src/components/assets/outgoing-traffic.svg b/ui/src/components/assets/outgoing-traffic.svg deleted file mode 100644 index 1c8ec87d7..000000000 --- a/ui/src/components/assets/outgoing-traffic.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ui/src/components/style/HarEntry.module.sass b/ui/src/components/style/HarEntry.module.sass index 3e28bfee9..fc5442f48 100644 --- a/ui/src/components/style/HarEntry.module.sass +++ b/ui/src/components/style/HarEntry.module.sass @@ -37,9 +37,10 @@ .timestamp font-size: 12px color: $secondary-font-color - padding-left: 8px - padding-right: 8px + padding-left: 12px flex-shrink: 0 + width: 145px + text-align: left .endpointServiceContainer display: flex @@ -51,13 +52,6 @@ .directionContainer display: flex - width: 28px - flex-direction: column - -.outgoingIcon - display: flex - align-self: flex-end - -.ingoingIcon - display: flex - align-self: flex-start + border-right: 1px solid $data-background-color + padding: 4px + padding-right: 12px From 3662fbcdf6e1b3425468ce27cc635e716772007a Mon Sep 17 00:00:00 2001 From: gadotroee <55343099+gadotroee@users.noreply.github.com> Date: Thu, 1 Jul 2021 16:18:02 +0300 Subject: [PATCH 06/12] Mizu analyze improvements (#90) --- api/pkg/api/socket_server_handlers.go | 5 +- api/pkg/controllers/entries_controller.go | 122 +------- api/pkg/controllers/status_controller.go | 5 + api/pkg/database/main.go | 45 +++ api/pkg/models/models.go | 2 - api/pkg/routes/public_routes.go | 1 + api/pkg/up9/main.go | 178 +++++++++++ cli/cmd/tap.go | 2 +- cli/cmd/tapRunner.go | 38 +-- cli/mizu/consts.go | 11 + shared/models.go | 32 +- ui/src/App.sass | 2 + ui/src/App.tsx | 47 ++- ui/src/components/HarPage.tsx | 37 ++- ui/src/style/mui.theme.ts | 366 ++++++++++++++++++++++ 15 files changed, 722 insertions(+), 171 deletions(-) create mode 100644 api/pkg/up9/main.go create mode 100644 ui/src/style/mui.theme.ts diff --git a/api/pkg/api/socket_server_handlers.go b/api/pkg/api/socket_server_handlers.go index 8164df004..1033dc7a8 100644 --- a/api/pkg/api/socket_server_handlers.go +++ b/api/pkg/api/socket_server_handlers.go @@ -9,6 +9,7 @@ import ( "mizuserver/pkg/controllers" "mizuserver/pkg/models" "mizuserver/pkg/routes" + "mizuserver/pkg/up9" ) var browserClientSocketUUIDs = make([]string, 0) @@ -18,6 +19,9 @@ type RoutesEventHandlers struct { SocketHarOutChannel chan<- *tap.OutputChannelItem } +func init() { + go up9.UpdateAnalyzeStatus(broadcastToBrowserClients) +} func (h *RoutesEventHandlers) WebSocketConnect(ep *ikisocket.EventPayload) { if ep.Kws.GetAttribute("is_tapper") == true { @@ -84,7 +88,6 @@ func (h *RoutesEventHandlers) WebSocketMessage(ep *ikisocket.EventPayload) { } } - func removeSocketUUIDFromBrowserSlice(uuidToRemove string) { newUUIDSlice := make([]string, 0, len(browserClientSocketUUIDs)) for _, uuid := range browserClientSocketUUIDs { diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index fa1306c7f..89dd450c2 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -1,41 +1,18 @@ package controllers import ( - "bytes" - "compress/zlib" "encoding/json" "fmt" "github.com/gofiber/fiber/v2" "github.com/google/martian/har" - "io/ioutil" - "log" "mizuserver/pkg/database" "mizuserver/pkg/models" + "mizuserver/pkg/up9" "mizuserver/pkg/utils" "mizuserver/pkg/validation" - "net/http" - "net/url" "time" ) -const ( - OrderDesc = "desc" - OrderAsc = "asc" - LT = "lt" - GT = "gt" -) - -var ( - operatorToSymbolMapping = map[string]string{ - LT: "<", - GT: ">", - } - operatorToOrderMapping = map[string]string{ - LT: OrderDesc, - GT: OrderAsc, - } -) - func GetEntries(c *fiber.Ctx) error { entriesFilter := &models.EntriesFilter{} @@ -47,8 +24,8 @@ func GetEntries(c *fiber.Ctx) error { return c.Status(fiber.StatusBadRequest).JSON(err) } - order := operatorToOrderMapping[entriesFilter.Operator] - operatorSymbol := operatorToSymbolMapping[entriesFilter.Operator] + order := database.OperatorToOrderMapping[entriesFilter.Operator] + operatorSymbol := database.OperatorToSymbolMapping[entriesFilter.Operator] var entries []models.MizuEntry database.GetEntriesTable(). Order(fmt.Sprintf("timestamp %s", order)). @@ -57,7 +34,7 @@ func GetEntries(c *fiber.Ctx) error { Limit(entriesFilter.Limit). Find(&entries) - if len(entries) > 0 && order == OrderDesc { + if len(entries) > 0 && order == database.OrderDesc { // the entries always order from oldest to newest so we should revers utils.ReverseSlice(entries) } @@ -73,7 +50,7 @@ func GetEntries(c *fiber.Ctx) error { func GetHARs(c *fiber.Ctx) error { entriesFilter := &models.HarFetchRequestBody{} - order := OrderDesc + order := database.OrderDesc if err := c.QueryParser(entriesFilter); err != nil { return c.Status(fiber.StatusBadRequest).JSON(err) } @@ -144,67 +121,20 @@ func GetHARs(c *fiber.Ctx) error { return c.Status(fiber.StatusOK).SendStream(buffer) } -func uploadEntriesImpl(token string, model string, envPrefix string) { - sleepTime := time.Second * 10 - - var timestampFrom int64 = 0 - - for { - timestampTo := time.Now().UnixNano() / int64(time.Millisecond) - fmt.Printf("Getting entries from %v, to %v\n", timestampFrom, timestampTo) - entriesArray := getEntriesFromDb(timestampFrom, timestampTo) - - if len(entriesArray) > 0 { - fmt.Printf("About to upload %v entries\n", len(entriesArray)) - - body, jMarshalErr := json.Marshal(entriesArray) - if jMarshalErr != nil { - log.Fatal(jMarshalErr) - } - - var in bytes.Buffer - w := zlib.NewWriter(&in) - _, _ = w.Write(body) - _ = w.Close() - reqBody := ioutil.NopCloser(bytes.NewReader(in.Bytes())) - - postUrl, _ := url.Parse(fmt.Sprintf("https://traffic.%s/dumpTrafficBulk/%s", envPrefix, model)) - fmt.Println(postUrl) - req := &http.Request{ - Method: http.MethodPost, - URL: postUrl, - Header: map[string][]string{ - "Content-Encoding": {"deflate"}, - "Content-Type": {"application/octet-stream"}, - "Guest-Auth": {token}, - }, - Body: reqBody, - } - _, postErr := http.DefaultClient.Do(req) - if postErr != nil { - log.Fatal(postErr) - } - fmt.Printf("Finish uploading %v entries to %s\n", len(entriesArray), postUrl) - - } else { - fmt.Println("Nothing to upload") - } - - fmt.Printf("Sleeping for %v...\n", sleepTime) - time.Sleep(sleepTime) - timestampFrom = timestampTo - } -} - func UploadEntries(c *fiber.Ctx) error { - entriesFilter := &models.UploadEntriesRequestBody{} - if err := c.QueryParser(entriesFilter); err != nil { + uploadRequestBody := &models.UploadEntriesRequestBody{} + if err := c.QueryParser(uploadRequestBody); err != nil { return c.Status(fiber.StatusBadRequest).JSON(err) } - if err := validation.Validate(entriesFilter); err != nil { + if err := validation.Validate(uploadRequestBody); err != nil { return c.Status(fiber.StatusBadRequest).JSON(err) } - go uploadEntriesImpl(entriesFilter.Token, entriesFilter.Model, entriesFilter.Dest) + if up9.GetAnalyzeInfo().IsAnalyzing { + return c.Status(fiber.StatusBadRequest).SendString("Cannot analyze, mizu is already analyzing") + } + + token, _ := up9.CreateAnonymousToken(uploadRequestBody.Dest) + go up9.UploadEntriesImpl(token.Token, token.Model, uploadRequestBody.Dest) return c.Status(fiber.StatusOK).SendString("OK") } @@ -231,32 +161,10 @@ func GetFullEntries(c *fiber.Ctx) error { timestampTo = entriesFilter.To } - entriesArray := getEntriesFromDb(timestampFrom, timestampTo) + entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo) return c.Status(fiber.StatusOK).JSON(entriesArray) } -func getEntriesFromDb(timestampFrom int64, timestampTo int64) []har.Entry { - order := OrderDesc - var entries []models.MizuEntry - database.GetEntriesTable(). - Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)). - Order(fmt.Sprintf("timestamp %s", order)). - Find(&entries) - - if len(entries) > 0 { - // the entries always order from oldest to newest so we should revers - utils.ReverseSlice(entries) - } - - entriesArray := make([]har.Entry, 0) - for _, entryData := range entries { - var harEntry har.Entry - _ = json.Unmarshal([]byte(entryData.Entry), &harEntry) - entriesArray = append(entriesArray, harEntry) - } - return entriesArray -} - func GetEntry(c *fiber.Ctx) error { var entryData models.EntryData database.GetEntriesTable(). diff --git a/api/pkg/controllers/status_controller.go b/api/pkg/controllers/status_controller.go index 15f30a67b..e4fed4c55 100644 --- a/api/pkg/controllers/status_controller.go +++ b/api/pkg/controllers/status_controller.go @@ -3,6 +3,7 @@ package controllers import ( "github.com/gofiber/fiber/v2" "github.com/up9inc/mizu/shared" + "mizuserver/pkg/up9" ) var TapStatus shared.TapStatus @@ -10,3 +11,7 @@ var TapStatus shared.TapStatus func GetTappingStatus(c *fiber.Ctx) error { return c.Status(fiber.StatusOK).JSON(TapStatus) } + +func AnalyzeInformation(c *fiber.Ctx) error { + return c.Status(fiber.StatusOK).JSON(up9.GetAnalyzeInfo()) +} diff --git a/api/pkg/database/main.go b/api/pkg/database/main.go index b015a0868..4ce4f07db 100644 --- a/api/pkg/database/main.go +++ b/api/pkg/database/main.go @@ -1,9 +1,13 @@ package database import ( + "encoding/json" + "fmt" + "github.com/google/martian/har" "gorm.io/driver/sqlite" "gorm.io/gorm" "mizuserver/pkg/models" + "mizuserver/pkg/utils" ) const ( @@ -14,6 +18,24 @@ var ( DB = initDataBase(DBPath) ) +const ( + OrderDesc = "desc" + OrderAsc = "asc" + LT = "lt" + GT = "gt" +) + +var ( + OperatorToSymbolMapping = map[string]string{ + LT: "<", + GT: ">", + } + OperatorToOrderMapping = map[string]string{ + LT: OrderDesc, + GT: OrderAsc, + } +) + func GetEntriesTable() *gorm.DB { return DB.Table("mizu_entries") } @@ -23,3 +45,26 @@ func initDataBase(databasePath string) *gorm.DB { _ = temp.AutoMigrate(&models.MizuEntry{}) // this will ensure table is created return temp } + +func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []har.Entry { + order := OrderDesc + var entries []models.MizuEntry + GetEntriesTable(). + Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)). + Order(fmt.Sprintf("timestamp %s", order)). + Find(&entries) + + if len(entries) > 0 { + // the entries always order from oldest to newest so we should revers + utils.ReverseSlice(entries) + } + + entriesArray := make([]har.Entry, 0) + for _, entryData := range entries { + var harEntry har.Entry + _ = json.Unmarshal([]byte(entryData.Entry), &harEntry) + entriesArray = append(entriesArray, harEntry) + } + return entriesArray +} + diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index b36ea7528..a45124255 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -50,8 +50,6 @@ type EntriesFilter struct { } type UploadEntriesRequestBody struct { - Token string `query:"token"` - Model string `query:"model"` Dest string `query:"dest"` } diff --git a/api/pkg/routes/public_routes.go b/api/pkg/routes/public_routes.go index ae78d96db..3bdbe150a 100644 --- a/api/pkg/routes/public_routes.go +++ b/api/pkg/routes/public_routes.go @@ -20,4 +20,5 @@ func EntriesRoutes(fiberApp *fiber.App) { routeGroup.Get("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB routeGroup.Get("/tapStatus", controllers.GetTappingStatus) // get tapping status + routeGroup.Get("/analyzeStatus", controllers.AnalyzeInformation) } diff --git a/api/pkg/up9/main.go b/api/pkg/up9/main.go new file mode 100644 index 000000000..17f1ab131 --- /dev/null +++ b/api/pkg/up9/main.go @@ -0,0 +1,178 @@ +package up9 + +import ( + "bytes" + "compress/zlib" + "encoding/json" + "fmt" + "github.com/up9inc/mizu/shared" + "io/ioutil" + "log" + "mizuserver/pkg/database" + "net/http" + "net/url" + "time" +) + + +const ( + AnalyzeCheckSleepTime = 5 * time.Second +) + +type GuestToken struct { + Token string `json:"token"` + Model string `json:"model"` +} + +type ModelStatus struct { + LastMajorGeneration float64 `json:"lastMajorGeneration"` +} + +func getGuestToken(url string, target *GuestToken) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + return json.NewDecoder(resp.Body).Decode(target) +} + +func CreateAnonymousToken(envPrefix string) (*GuestToken, error) { + tokenUrl := fmt.Sprintf("https://trcc.%v/anonymous/token", envPrefix) + token := &GuestToken{} + if err := getGuestToken(tokenUrl, token); err != nil { + fmt.Println(err) + return nil, err + } + return token, nil +} + +func GetRemoteUrl(analyzeDestination string, analyzeToken string) string { + return fmt.Sprintf("https://%s/share/%s", analyzeDestination, analyzeToken) +} + +func CheckIfModelReady(analyzeDestination string, analyzeModel string, analyzeToken string) bool { + statusUrl, _ := url.Parse(fmt.Sprintf("https://trcc.%s/models/%s/status", analyzeDestination, analyzeModel)) + req := &http.Request{ + Method: http.MethodGet, + URL: statusUrl, + Header: map[string][]string{ + "Content-Type": {"application/json"}, + "Guest-Auth": {analyzeToken}, + }, + } + statusResp, err := http.DefaultClient.Do(req) + if err != nil { + return false + } + defer statusResp.Body.Close() + + target := &ModelStatus{} + _ = json.NewDecoder(statusResp.Body).Decode(&target) + + return target.LastMajorGeneration > 0 +} + +func GetTrafficDumpUrl(analyzeDestination string, analyzeModel string) *url.URL { + postUrl, _ := url.Parse(fmt.Sprintf("https://traffic.%s/dumpTrafficBulk/%s", analyzeDestination, analyzeModel)) + return postUrl +} + +type AnalyzeInformation struct { + IsAnalyzing bool + AnalyzedModel string + AnalyzeToken string + AnalyzeDestination string +} + +func (info *AnalyzeInformation) Reset() { + info.IsAnalyzing = false + info.AnalyzedModel = "" + info.AnalyzeToken = "" + info.AnalyzeDestination = "" +} + +var analyzeInformation = &AnalyzeInformation{} + +func GetAnalyzeInfo() *shared.AnalyzeStatus { + return &shared.AnalyzeStatus{ + IsAnalyzing: analyzeInformation.IsAnalyzing, + RemoteUrl: GetRemoteUrl(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzeToken), + IsRemoteReady: CheckIfModelReady(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzedModel, analyzeInformation.AnalyzeToken), + } +} + +func UploadEntriesImpl(token string, model string, envPrefix string) { + analyzeInformation.IsAnalyzing = true + analyzeInformation.AnalyzedModel = model + analyzeInformation.AnalyzeToken = token + analyzeInformation.AnalyzeDestination = envPrefix + + sleepTime := time.Second * 10 + + var timestampFrom int64 = 0 + + for { + timestampTo := time.Now().UnixNano() / int64(time.Millisecond) + fmt.Printf("Getting entries from %v, to %v\n", timestampFrom, timestampTo) + entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo) + + if len(entriesArray) > 0 { + fmt.Printf("About to upload %v entries\n", len(entriesArray)) + + body, jMarshalErr := json.Marshal(entriesArray) + if jMarshalErr != nil { + analyzeInformation.Reset() + fmt.Println("Stopping analyzing") + log.Fatal(jMarshalErr) + } + + var in bytes.Buffer + w := zlib.NewWriter(&in) + _, _ = w.Write(body) + _ = w.Close() + reqBody := ioutil.NopCloser(bytes.NewReader(in.Bytes())) + + req := &http.Request{ + Method: http.MethodPost, + URL: GetTrafficDumpUrl(envPrefix, model), + Header: map[string][]string{ + "Content-Encoding": {"deflate"}, + "Content-Type": {"application/octet-stream"}, + "Guest-Auth": {token}, + }, + Body: reqBody, + } + + if _, postErr := http.DefaultClient.Do(req); postErr != nil { + analyzeInformation.Reset() + log.Println("Stopping analyzing") + log.Fatal(postErr) + } + fmt.Printf("Finish uploading %v entries to %s\n", len(entriesArray), GetTrafficDumpUrl(envPrefix, model)) + + } else { + fmt.Println("Nothing to upload") + } + + fmt.Printf("Sleeping for %v...\n", sleepTime) + time.Sleep(sleepTime) + timestampFrom = timestampTo + } +} + +func UpdateAnalyzeStatus(callback func(data []byte)) { + for { + if !analyzeInformation.IsAnalyzing { + time.Sleep(AnalyzeCheckSleepTime) + continue + } + analyzeStatus := GetAnalyzeInfo() + socketMessage := shared.CreateWebSocketMessageTypeAnalyzeStatus(*analyzeStatus) + + jsonMessage, _ := json.Marshal(socketMessage) + callback(jsonMessage) + time.Sleep(AnalyzeCheckSleepTime) + } +} diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index c23afd56f..f29a59773 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -63,7 +63,7 @@ func init() { tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver") tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector") - tapCmd.Flags().BoolVar(&mizuTapOptions.Analyze, "analyze", false, "Uploads traffic to UP9 cloud for further analysis") + tapCmd.Flags().BoolVar(&mizuTapOptions.Analyze, "analyze", false, "Uploads traffic to UP9 cloud for further analysis (Beta)") tapCmd.Flags().StringVar(&mizuTapOptions.AnalyzeDestination, "dest", "up9.app", "Destination environment") tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces") tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file") diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 7019c4715..dbb373a2a 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "encoding/json" "fmt" "net/http" "os" @@ -81,30 +80,6 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { waitForFinish(ctx, cancel) } -type GuestToken struct { - Token string `json:"token"` - Model string `json:"model"` -} - -func getGuestToken(url string, target *GuestToken) error { - resp, err := http.Get(url) - if err != nil { - return err - } - defer resp.Body.Close() - - return json.NewDecoder(resp.Body).Decode(target) -} - -func CreateAnonymousToken(envPrefix string) (*GuestToken, error) { - tokenUrl := fmt.Sprintf("https://trcc.%v/anonymous/token", envPrefix) - token := &GuestToken{} - if err := getGuestToken(tokenUrl, token); err != nil { - fmt.Println(err) - return nil, err - } - return token, nil -} func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error { if err := createMizuAggregator(ctx, kubernetesProvider, tappingOptions, mizuApiFilteringOptions); err != nil { @@ -225,10 +200,10 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro for { select { case newTarget := <-added: - fmt.Printf("+%s\n", newTarget.Name) + fmt.Printf(mizu.Green, fmt.Sprintf("+%s\n", newTarget.Name)) case removedTarget := <-removed: - fmt.Printf("-%s\n", removedTarget.Name) + fmt.Printf(mizu.Red, fmt.Sprintf("-%s\n", removedTarget.Name)) restartTappersDebouncer.SetOn() case modifiedTarget := <-modified: @@ -274,14 +249,13 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi cancel() } else { fmt.Printf("Web interface is now available at http://localhost:%d\n", tappingOptions.GuiPort) - + time.Sleep(time.Second * 5) // Waiting to be sure port forwarding finished if tappingOptions.Analyze { - token, _ := CreateAnonymousToken(tappingOptions.AnalyzeDestination) - if _, err := http.Get(fmt.Sprintf("http://localhost:%d/api/uploadEntries?token=%s&model=%s&dest=%s", tappingOptions.GuiPort, token.Token, token.Model, tappingOptions.AnalyzeDestination)); err != nil { + if _, err := http.Get(fmt.Sprintf("http://localhost:%d/api/uploadEntries?dest=%s", tappingOptions.GuiPort, tappingOptions.AnalyzeDestination)); err != nil { fmt.Println(err) } else { - fmt.Println("Staring to upload and analyze the data, it may take a few minutes") - fmt.Println("https://" + tappingOptions.AnalyzeDestination + "/share/" + token.Token) + fmt.Printf(mizu.Yellow, "Traffic is uploading to UP9 cloud for further analsys") + fmt.Println() } } } diff --git a/cli/mizu/consts.go b/cli/mizu/consts.go index 9f72620ad..1f960ead2 100644 --- a/cli/mizu/consts.go +++ b/cli/mizu/consts.go @@ -15,3 +15,14 @@ const ( TapperPodName = "mizu-tapper" K8sAllNamespaces = "" ) + +const ( + Black = "\033[1;30m%s\033[0m" + Red = "\033[1;31m%s\033[0m" + Green = "\033[1;32m%s\033[0m" + Yellow = "\033[1;33m%s\033[0m" + Purple = "\033[1;34m%s\033[0m" + Magenta = "\033[1;35m%s\033[0m" + Teal = "\033[1;36m%s\033[0m" + White = "\033[1;37m%s\033[0m" +) diff --git a/shared/models.go b/shared/models.go index 854dbf1f6..f1d443cdf 100644 --- a/shared/models.go +++ b/shared/models.go @@ -1,28 +1,41 @@ package shared type WebSocketMessageType string + const ( - WebSocketMessageTypeEntry WebSocketMessageType = "entry" - WebSocketMessageTypeTappedEntry WebSocketMessageType = "tappedEntry" - WebSocketMessageTypeUpdateStatus WebSocketMessageType = "status" + WebSocketMessageTypeEntry WebSocketMessageType = "entry" + WebSocketMessageTypeTappedEntry WebSocketMessageType = "tappedEntry" + WebSocketMessageTypeUpdateStatus WebSocketMessageType = "status" + WebSocketMessageTypeAnalyzeStatus WebSocketMessageType = "analyzeStatus" ) type WebSocketMessageMetadata struct { MessageType WebSocketMessageType `json:"messageType,omitempty"` } +type WebSocketAnalyzeStatusMessage struct { + *WebSocketMessageMetadata + AnalyzeStatus AnalyzeStatus `json:"analyzeStatus"` +} + +type AnalyzeStatus struct { + IsAnalyzing bool `json:"isAnalyzing"` + RemoteUrl string `json:"remoteUrl"` + IsRemoteReady bool `json:"isRemoteReady"` +} + type WebSocketStatusMessage struct { *WebSocketMessageMetadata TappingStatus TapStatus `json:"tappingStatus"` } type TapStatus struct { - Pods []PodInfo `json:"pods"` + Pods []PodInfo `json:"pods"` } type PodInfo struct { Namespace string `json:"namespace"` - Name string `json:"name"` + Name string `json:"name"` } func CreateWebSocketStatusMessage(tappingStatus TapStatus) WebSocketStatusMessage { @@ -34,6 +47,15 @@ func CreateWebSocketStatusMessage(tappingStatus TapStatus) WebSocketStatusMessag } } +func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSocketAnalyzeStatusMessage { + return WebSocketAnalyzeStatusMessage{ + WebSocketMessageMetadata: &WebSocketMessageMetadata{ + MessageType: WebSocketMessageTypeAnalyzeStatus, + }, + AnalyzeStatus: analyzeStatus, + } +} + type TrafficFilteringOptions struct { PlainTextMaskingRegexes []*SerializableRegexp } diff --git a/ui/src/App.sass b/ui/src/App.sass index 564ee6c2b..ab781429e 100644 --- a/ui/src/App.sass +++ b/ui/src/App.sass @@ -10,6 +10,8 @@ display: flex align-items: center padding-left: 24px + padding-right: 24px + justify-content: space-between .title font-size: 45px diff --git a/ui/src/App.tsx b/ui/src/App.tsx index 4195e558d..470de200c 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -1,18 +1,43 @@ -import React from 'react'; -import {HarPage} from "./components/HarPage"; +import React, {useState} from 'react'; import './App.sass'; import logo from './components/assets/Mizu.svg'; +import {Button, ThemeProvider} from "@material-ui/core"; +import Theme from './style/mui.theme'; +import {HarPage} from "./components/HarPage"; + const App = () => { - return ( -
-
-
logo
-
Traffic viewer for Kubernetes
-
- -
- ); + + const [analyzeStatus, setAnalyzeStatus] = useState(null); + + return ( + +
+
+
+
logo
+
Traffic viewer for Kubernetes
+
+
+ {analyzeStatus?.isAnalyzing && +
+ +
+ } +
+
+ +
+ + ); } export default App; diff --git a/ui/src/components/HarPage.tsx b/ui/src/components/HarPage.tsx index 11edf80fa..6005cdb1b 100644 --- a/ui/src/components/HarPage.tsx +++ b/ui/src/components/HarPage.tsx @@ -35,7 +35,11 @@ enum ConnectionStatus { Paused } -export const HarPage: React.FC = () => { +interface HarPageProps { + setAnalyzeStatus: (status: any) => void; +} + +export const HarPage: React.FC = ({setAnalyzeStatus}) => { const classes = useLayoutStyles(); @@ -60,21 +64,21 @@ export const HarPage: React.FC = () => { ws.current.onclose = () => setConnection(ConnectionStatus.Closed); } - if(ws.current) { + if (ws.current) { ws.current.onmessage = e => { - if(!e?.data) return; + if (!e?.data) return; const message = JSON.parse(e.data); switch (message.messageType) { case "entry": const entry = message.data - if(connection === ConnectionStatus.Paused) { + if (connection === ConnectionStatus.Paused) { setNoMoreDataBottom(false) return; } - if(!focusedEntryId) setFocusedEntryId(entry.id) + if (!focusedEntryId) setFocusedEntryId(entry.id) let newEntries = [...entries]; - if(entries.length === 1000) { + if (entries.length === 1000) { newEntries = newEntries.splice(1); setNoMoreDataTop(false); } @@ -83,6 +87,9 @@ export const HarPage: React.FC = () => { case "status": setTappingStatus(message.tappingStatus); break + case "analyzeStatus": + setAnalyzeStatus(message.analyzeStatus); + break default: console.error(`unsupported websocket message type, Got: ${message.messageType}`) } @@ -94,19 +101,23 @@ export const HarPage: React.FC = () => { fetch(`http://localhost:8899/api/tapStatus`) .then(response => response.json()) .then(data => setTappingStatus(data)); + + fetch(`http://localhost:8899/api/analyzeStatus`) + .then(response => response.json()) + .then(data => setAnalyzeStatus(data)); }, []); useEffect(() => { - if(!focusedEntryId) return; + if (!focusedEntryId) return; setSelectedHarEntry(null) fetch(`http://localhost:8899/api/entries/${focusedEntryId}`) .then(response => response.json()) .then(data => setSelectedHarEntry(data)); - },[focusedEntryId]) + }, [focusedEntryId]) const toggleConnection = () => { - setConnection(connection === ConnectionStatus.Connected ? ConnectionStatus.Paused : ConnectionStatus.Connected ); + setConnection(connection === ConnectionStatus.Connected ? ConnectionStatus.Paused : ConnectionStatus.Connected); } const getConnectionStatusClass = (isContainer) => { @@ -135,11 +146,12 @@ export const HarPage: React.FC = () => { return (
- pause + pause
{getConnectionTitle()}
-
+
@@ -169,7 +181,8 @@ export const HarPage: React.FC = () => {
- {selectedHarEntry && } + {selectedHarEntry && + }
} {tappingStatus?.pods != null && } diff --git a/ui/src/style/mui.theme.ts b/ui/src/style/mui.theme.ts new file mode 100644 index 000000000..a9f24b8ad --- /dev/null +++ b/ui/src/style/mui.theme.ts @@ -0,0 +1,366 @@ +import createMuiTheme, { Theme, ThemeOptions } from "@material-ui/core/styles/createMuiTheme"; +import { Palette } from "@material-ui/core/styles/createPalette"; + +interface IColor { + main: string; + light: string; + dark: string; + contrastText: string; +} + +interface IPalette extends Palette { + primaryBackground: IColor + secondaryBackground: IColor +} + +export interface ITheme extends Theme { + palette: IPalette; +} + +interface IThemeOptions extends ThemeOptions { + palette: IPalette; + overrides: any; +} + +const theme = createMuiTheme({ + "palette": { + "common": { + "black": "#000", + "white": "#fff" + }, + "type": "dark", + "primary": { + "main": "#627ef7", + "light": "#a0b2ff", + "dark": "#2b3560", + "contrastText": "#090b14" + }, + "primaryBackground": { + "main": "#171c30", + "light": "#252c47", + "dark": "#090b14", + "contrastText": "#fff" + }, + "secondary": { + "main": "rgba(255, 255, 255, 0.75)", + "light": "#fff", + "dark": "rgba(255, 255, 255, 0.5)", + "contrastText": "#000" + }, + "secondaryBackground": { + "main": "rgba(255, 255, 255, 0.12)", + "light": "rgba(255, 255, 255, 0.25)", + "dark": "rgba(255, 255, 255, 0.06)", + "contrastText": "#fff" + }, + "error": { + "main": "#cb5411", + "light": "#ff3a30", + "dark": "rgba(255, 58, 48, 0.35)", + "contrastText": "#fff" + }, + "warning": { + "light": "#ffb74d", + "main": "#ff9800", + "dark": "#f57c00", + "contrastText": "#344073" + }, + "info": { + "light": "#64b5f6", + "main": "#2196f3", + "dark": "#1976d2", + "contrastText": "#fff" + }, + "success": { + "light": "#3eb545", + "main": "rgba(62, 181, 69, 0.5)", + "dark": "rgba(62, 181, 69, 0.35)", + "contrastText": "#fff" + }, + "grey": { + "50": "#fafafa", + "100": "#f5f5f5", + "200": "#eeeeee", + "300": "#e0e0e0", + "400": "#bdbdbd", + "500": "#9e9e9e", + "600": "#757575", + "700": "#616161", + "800": "#424242", + "900": "#212121", + "A100": "#d5d5d5", + "A200": "#aaaaaa", + "A400": "#303030", + "A700": "#616161" + }, + "contrastThreshold": 3, + "tonalOffset": 0.2, + "text": { + "primary": "#fff", + "secondary": "rgba(255, 255, 255, 0.85)", + "disabled": "rgba(255, 255, 255, 0.75)", + "hint": "rgba(255, 255, 255, 0.65)", + }, + "divider": "rgba(255, 255, 255, 0.12)", + "background": { + "paper": "#344073", + "default": "#252c47", + }, + "action": { + "active": "#fff", + "hover": "rgba(255, 255, 255, 0.08)", + "hoverOpacity": 0.08, + "selected": "rgba(255, 255, 255, 0.16)", + "selectedOpacity": 0.16, + "disabled": "rgba(255, 255, 255, 0.3)", + "disabledBackground": "rgba(255, 255, 255, 0.12)", + "disabledOpacity": 0.38, + "focus": "rgba(255, 255, 255, 0.12)", + "focusOpacity": 0.12, + "activatedOpacity": 0.24 + } + }, + "props": {}, + "shadows": [ + "none", + "0px 2px 1px -1px rgba(0,0,0,0.2),0px 1px 1px 0px rgba(0,0,0,0.14),0px 1px 3px 0px rgba(0,0,0,0.12)", + "0px 3px 1px -2px rgba(0,0,0,0.2),0px 2px 2px 0px rgba(0,0,0,0.14),0px 1px 5px 0px rgba(0,0,0,0.12)", + "0px 3px 3px -2px rgba(0,0,0,0.2),0px 3px 4px 0px rgba(0,0,0,0.14),0px 1px 8px 0px rgba(0,0,0,0.12)", + "0px 2px 4px -1px rgba(0,0,0,0.2),0px 4px 5px 0px rgba(0,0,0,0.14),0px 1px 10px 0px rgba(0,0,0,0.12)", + "0px 3px 5px -1px rgba(0,0,0,0.2),0px 5px 8px 0px rgba(0,0,0,0.14),0px 1px 14px 0px rgba(0,0,0,0.12)", + "0px 3px 5px -1px rgba(0,0,0,0.2),0px 6px 10px 0px rgba(0,0,0,0.14),0px 1px 18px 0px rgba(0,0,0,0.12)", + "0px 4px 5px -2px rgba(0,0,0,0.2),0px 7px 10px 1px rgba(0,0,0,0.14),0px 2px 16px 1px rgba(0,0,0,0.12)", + "0px 5px 5px -3px rgba(0,0,0,0.2),0px 8px 10px 1px rgba(0,0,0,0.14),0px 3px 14px 2px rgba(0,0,0,0.12)", + "0px 5px 6px -3px rgba(0,0,0,0.2),0px 9px 12px 1px rgba(0,0,0,0.14),0px 3px 16px 2px rgba(0,0,0,0.12)", + "0px 6px 6px -3px rgba(0,0,0,0.2),0px 10px 14px 1px rgba(0,0,0,0.14),0px 4px 18px 3px rgba(0,0,0,0.12)", + "0px 6px 7px -4px rgba(0,0,0,0.2),0px 11px 15px 1px rgba(0,0,0,0.14),0px 4px 20px 3px rgba(0,0,0,0.12)", + "0px 7px 8px -4px rgba(0,0,0,0.2),0px 12px 17px 2px rgba(0,0,0,0.14),0px 5px 22px 4px rgba(0,0,0,0.12)", + "0px 7px 8px -4px rgba(0,0,0,0.2),0px 13px 19px 2px rgba(0,0,0,0.14),0px 5px 24px 4px rgba(0,0,0,0.12)", + "0px 7px 9px -4px rgba(0,0,0,0.2),0px 14px 21px 2px rgba(0,0,0,0.14),0px 5px 26px 4px rgba(0,0,0,0.12)", + "0px 8px 9px -5px rgba(0,0,0,0.2),0px 15px 22px 2px rgba(0,0,0,0.14),0px 6px 28px 5px rgba(0,0,0,0.12)", + "0px 8px 10px -5px rgba(0,0,0,0.2),0px 16px 24px 2px rgba(0,0,0,0.14),0px 6px 30px 5px rgba(0,0,0,0.12)", + "0px 8px 11px -5px rgba(0,0,0,0.2),0px 17px 26px 2px rgba(0,0,0,0.14),0px 6px 32px 5px rgba(0,0,0,0.12)", + "0px 9px 11px -5px rgba(0,0,0,0.2),0px 18px 28px 2px rgba(0,0,0,0.14),0px 7px 34px 6px rgba(0,0,0,0.12)", + "0px 9px 12px -6px rgba(0,0,0,0.2),0px 19px 29px 2px rgba(0,0,0,0.14),0px 7px 36px 6px rgba(0,0,0,0.12)", + "0px 10px 13px -6px rgba(0,0,0,0.2),0px 20px 31px 3px rgba(0,0,0,0.14),0px 8px 38px 7px rgba(0,0,0,0.12)", + "0px 10px 13px -6px rgba(0,0,0,0.2),0px 21px 33px 3px rgba(0,0,0,0.14),0px 8px 40px 7px rgba(0,0,0,0.12)", + "0px 10px 14px -6px rgba(0,0,0,0.2),0px 22px 35px 3px rgba(0,0,0,0.14),0px 8px 42px 7px rgba(0,0,0,0.12)", + "0px 11px 14px -7px rgba(0,0,0,0.2),0px 23px 36px 3px rgba(0,0,0,0.14),0px 9px 44px 8px rgba(0,0,0,0.12)", + "0px 11px 15px -7px rgba(0,0,0,0.2),0px 24px 38px 3px rgba(0,0,0,0.14),0px 9px 46px 8px rgba(0,0,0,0.12)" + ], + "typography": { + "htmlFontSize": 16, + "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", + "fontSize": 14, + "fontWeightLight": 300, + "fontWeightRegular": 400, + "fontWeightMedium": 500, + "fontWeightBold": 600, + "h1": { + "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", + "fontWeight": 300, + "fontSize": "6rem", + "lineHeight": 1.167, + "letterSpacing": "-0.01562em" + }, + "h2": { + "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", + "fontWeight": 300, + "fontSize": "3.75rem", + "lineHeight": 1.2, + "letterSpacing": "-0.00833em" + }, + "h3": { + "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", + "fontWeight": 400, + "fontSize": "3rem", + "lineHeight": 1.167, + "letterSpacing": "0em" + }, + "h4": { + "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", + "fontWeight": 400, + "fontSize": "2.125rem", + "lineHeight": 1.235, + "letterSpacing": "0.00735em" + }, + "h5": { + "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", + "fontWeight": 400, + "fontSize": "1.2rem", + "lineHeight": 1.334, + "letterSpacing": "0.02em" + }, + "h6": { + "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", + "fontWeight": 400, + "fontSize": "1rem", + "lineHeight": 1.6, + "letterSpacing": "0.0075em", + "textTransform": "uppercase" + }, + "subtitle1": { + "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", + "fontWeight": 400, + "fontSize": "1rem", + "lineHeight": 1.75, + "letterSpacing": "0.00938em" + }, + "subtitle2": { + "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", + "fontWeight": 500, + "fontSize": "0.875rem", + "lineHeight": 1.57, + "letterSpacing": "0.00714em" + }, + "body1": { + "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", + "fontWeight": 400, + "fontSize": "1rem", + "lineHeight": 1.5, + "letterSpacing": "0.00938em" + }, + "body2": { + "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", + "fontWeight": 400, + "fontSize": "0.875rem", + "lineHeight": 1.43, + "letterSpacing": "0.01071em" + }, + "button": { + "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", + "fontWeight": 500, + "fontSize": "0.875rem", + "lineHeight": 1.75, + "letterSpacing": "0.02857em", + "textTransform": "uppercase" + }, + "caption": { + "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", + "fontWeight": 400, + "fontSize": "0.8rem", + "lineHeight": 1.66, + "letterSpacing": "0.03333em" + }, + "overline": { + "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", + "fontWeight": 400, + "fontSize": "0.75rem", + "lineHeight": 2.66, + "letterSpacing": "0.08333em", + "textTransform": "uppercase" + } + }, + "shape": { + "borderRadius": 16 + }, + "transitions": { + "easing": { + "easeInOut": "cubic-bezier(0.4, 0, 0.2, 1)", + "easeOut": "cubic-bezier(0.0, 0, 0.2, 1)", + "easeIn": "cubic-bezier(0.4, 0, 1, 1)", + "sharp": "cubic-bezier(0.4, 0, 0.6, 1)" + }, + "duration": { + "shortest": 150, + "shorter": 200, + "short": 250, + "standard": 300, + "complex": 375, + "enteringScreen": 225, + "leavingScreen": 195 + } + }, + "zIndex": { + "mobileStepper": 1000, + "speedDial": 1050, + "appBar": 1100, + "drawer": 1200, + "modal": 1300, + "snackbar": 1400, + "tooltip": 1500 + } +} as IThemeOptions) as ITheme; + +theme.overrides = { + + ...theme.overrides, + + MuiTooltip: { + tooltip: { + backgroundColor: "#2a335a", + fontSize: "14px", + borderRadius: "8px", + } + }, + + "MuiButton": { + label: { + fontWeight: "bold", + }, + "contained": { + color: '#F7F9FC', + padding: "7px 18px", + }, + "containedPrimary": { + + color: '#F7F9FC', + + "&:disabled": { + backgroundColor: "rgba(0,0,0,0.15)", + color: "rgba(255, 255, 255, 0.8)" + } + }, + "outlinedPrimary": { + + padding: "8px 18px", + "&:hover": { + backgroundColor: "rgba(255, 255, 255, 0.06)" + }, + + "&:disabled": { + backgroundColor: "rgba(255, 255, 255, 0.06)", + color: "rgba(255, 255, 255, 0.25)" + }, + + } + }, + + "MuiSwitch": { + root: { + padding: 11, + width: 44, + height: 29, + }, + switchBase: { + '&$checked': { + transform: "translateX(10px)", + "& $thumb": { + backgroundColor: theme.palette.common.white, + }, + "&.MuiSwitch-colorPrimary + $track": { + backgroundColor: theme.palette.secondaryBackground.dark, + border: "3px solid " + theme.palette.primary.light, + opacity: 1, + }, + } + }, + thumb: { + width: 8, + height: 8, + border: "3px solid " + theme.palette.primary.main, + backgroundColor: theme.palette.primary.dark, + boxShadow: 'none', + }, + track: { + width: 14, + height: 4, + backgroundColor: theme.palette.secondaryBackground.dark, + border: "3px solid " + theme.palette.primary.light, + opacity: 1, + } + } +}; + +export default theme; + From fc03ba2eda950dd3ef8af1b701e083e3d17d5196 Mon Sep 17 00:00:00 2001 From: gadotroee <55343099+gadotroee@users.noreply.github.com> Date: Thu, 1 Jul 2021 23:04:35 +0300 Subject: [PATCH 07/12] Fix styles (#91) --- cli/cmd/tapRunner.go | 2 +- ui/src/App.tsx | 50 +++--- ui/src/style/mui.theme.ts | 366 -------------------------------------- 3 files changed, 25 insertions(+), 393 deletions(-) delete mode 100644 ui/src/style/mui.theme.ts diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index dbb373a2a..5a9039daf 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -254,7 +254,7 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi if _, err := http.Get(fmt.Sprintf("http://localhost:%d/api/uploadEntries?dest=%s", tappingOptions.GuiPort, tappingOptions.AnalyzeDestination)); err != nil { fmt.Println(err) } else { - fmt.Printf(mizu.Yellow, "Traffic is uploading to UP9 cloud for further analsys") + fmt.Printf(mizu.Purple, "Traffic is uploading to UP9 cloud for further analsys") fmt.Println() } } diff --git a/ui/src/App.tsx b/ui/src/App.tsx index 470de200c..0663cfbb6 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -1,8 +1,7 @@ import React, {useState} from 'react'; import './App.sass'; import logo from './components/assets/Mizu.svg'; -import {Button, ThemeProvider} from "@material-ui/core"; -import Theme from './style/mui.theme'; +import {Button} from "@material-ui/core"; import {HarPage} from "./components/HarPage"; @@ -11,32 +10,31 @@ const App = () => { const [analyzeStatus, setAnalyzeStatus] = useState(null); return ( - -
-
-
-
logo
-
Traffic viewer for Kubernetes
-
-
- {analyzeStatus?.isAnalyzing && -
- -
- } -
+
+
+
+
logo
+
Traffic viewer for Kubernetes
+
+
+ {analyzeStatus?.isAnalyzing && +
+ +
+ }
-
- + +
); } diff --git a/ui/src/style/mui.theme.ts b/ui/src/style/mui.theme.ts deleted file mode 100644 index a9f24b8ad..000000000 --- a/ui/src/style/mui.theme.ts +++ /dev/null @@ -1,366 +0,0 @@ -import createMuiTheme, { Theme, ThemeOptions } from "@material-ui/core/styles/createMuiTheme"; -import { Palette } from "@material-ui/core/styles/createPalette"; - -interface IColor { - main: string; - light: string; - dark: string; - contrastText: string; -} - -interface IPalette extends Palette { - primaryBackground: IColor - secondaryBackground: IColor -} - -export interface ITheme extends Theme { - palette: IPalette; -} - -interface IThemeOptions extends ThemeOptions { - palette: IPalette; - overrides: any; -} - -const theme = createMuiTheme({ - "palette": { - "common": { - "black": "#000", - "white": "#fff" - }, - "type": "dark", - "primary": { - "main": "#627ef7", - "light": "#a0b2ff", - "dark": "#2b3560", - "contrastText": "#090b14" - }, - "primaryBackground": { - "main": "#171c30", - "light": "#252c47", - "dark": "#090b14", - "contrastText": "#fff" - }, - "secondary": { - "main": "rgba(255, 255, 255, 0.75)", - "light": "#fff", - "dark": "rgba(255, 255, 255, 0.5)", - "contrastText": "#000" - }, - "secondaryBackground": { - "main": "rgba(255, 255, 255, 0.12)", - "light": "rgba(255, 255, 255, 0.25)", - "dark": "rgba(255, 255, 255, 0.06)", - "contrastText": "#fff" - }, - "error": { - "main": "#cb5411", - "light": "#ff3a30", - "dark": "rgba(255, 58, 48, 0.35)", - "contrastText": "#fff" - }, - "warning": { - "light": "#ffb74d", - "main": "#ff9800", - "dark": "#f57c00", - "contrastText": "#344073" - }, - "info": { - "light": "#64b5f6", - "main": "#2196f3", - "dark": "#1976d2", - "contrastText": "#fff" - }, - "success": { - "light": "#3eb545", - "main": "rgba(62, 181, 69, 0.5)", - "dark": "rgba(62, 181, 69, 0.35)", - "contrastText": "#fff" - }, - "grey": { - "50": "#fafafa", - "100": "#f5f5f5", - "200": "#eeeeee", - "300": "#e0e0e0", - "400": "#bdbdbd", - "500": "#9e9e9e", - "600": "#757575", - "700": "#616161", - "800": "#424242", - "900": "#212121", - "A100": "#d5d5d5", - "A200": "#aaaaaa", - "A400": "#303030", - "A700": "#616161" - }, - "contrastThreshold": 3, - "tonalOffset": 0.2, - "text": { - "primary": "#fff", - "secondary": "rgba(255, 255, 255, 0.85)", - "disabled": "rgba(255, 255, 255, 0.75)", - "hint": "rgba(255, 255, 255, 0.65)", - }, - "divider": "rgba(255, 255, 255, 0.12)", - "background": { - "paper": "#344073", - "default": "#252c47", - }, - "action": { - "active": "#fff", - "hover": "rgba(255, 255, 255, 0.08)", - "hoverOpacity": 0.08, - "selected": "rgba(255, 255, 255, 0.16)", - "selectedOpacity": 0.16, - "disabled": "rgba(255, 255, 255, 0.3)", - "disabledBackground": "rgba(255, 255, 255, 0.12)", - "disabledOpacity": 0.38, - "focus": "rgba(255, 255, 255, 0.12)", - "focusOpacity": 0.12, - "activatedOpacity": 0.24 - } - }, - "props": {}, - "shadows": [ - "none", - "0px 2px 1px -1px rgba(0,0,0,0.2),0px 1px 1px 0px rgba(0,0,0,0.14),0px 1px 3px 0px rgba(0,0,0,0.12)", - "0px 3px 1px -2px rgba(0,0,0,0.2),0px 2px 2px 0px rgba(0,0,0,0.14),0px 1px 5px 0px rgba(0,0,0,0.12)", - "0px 3px 3px -2px rgba(0,0,0,0.2),0px 3px 4px 0px rgba(0,0,0,0.14),0px 1px 8px 0px rgba(0,0,0,0.12)", - "0px 2px 4px -1px rgba(0,0,0,0.2),0px 4px 5px 0px rgba(0,0,0,0.14),0px 1px 10px 0px rgba(0,0,0,0.12)", - "0px 3px 5px -1px rgba(0,0,0,0.2),0px 5px 8px 0px rgba(0,0,0,0.14),0px 1px 14px 0px rgba(0,0,0,0.12)", - "0px 3px 5px -1px rgba(0,0,0,0.2),0px 6px 10px 0px rgba(0,0,0,0.14),0px 1px 18px 0px rgba(0,0,0,0.12)", - "0px 4px 5px -2px rgba(0,0,0,0.2),0px 7px 10px 1px rgba(0,0,0,0.14),0px 2px 16px 1px rgba(0,0,0,0.12)", - "0px 5px 5px -3px rgba(0,0,0,0.2),0px 8px 10px 1px rgba(0,0,0,0.14),0px 3px 14px 2px rgba(0,0,0,0.12)", - "0px 5px 6px -3px rgba(0,0,0,0.2),0px 9px 12px 1px rgba(0,0,0,0.14),0px 3px 16px 2px rgba(0,0,0,0.12)", - "0px 6px 6px -3px rgba(0,0,0,0.2),0px 10px 14px 1px rgba(0,0,0,0.14),0px 4px 18px 3px rgba(0,0,0,0.12)", - "0px 6px 7px -4px rgba(0,0,0,0.2),0px 11px 15px 1px rgba(0,0,0,0.14),0px 4px 20px 3px rgba(0,0,0,0.12)", - "0px 7px 8px -4px rgba(0,0,0,0.2),0px 12px 17px 2px rgba(0,0,0,0.14),0px 5px 22px 4px rgba(0,0,0,0.12)", - "0px 7px 8px -4px rgba(0,0,0,0.2),0px 13px 19px 2px rgba(0,0,0,0.14),0px 5px 24px 4px rgba(0,0,0,0.12)", - "0px 7px 9px -4px rgba(0,0,0,0.2),0px 14px 21px 2px rgba(0,0,0,0.14),0px 5px 26px 4px rgba(0,0,0,0.12)", - "0px 8px 9px -5px rgba(0,0,0,0.2),0px 15px 22px 2px rgba(0,0,0,0.14),0px 6px 28px 5px rgba(0,0,0,0.12)", - "0px 8px 10px -5px rgba(0,0,0,0.2),0px 16px 24px 2px rgba(0,0,0,0.14),0px 6px 30px 5px rgba(0,0,0,0.12)", - "0px 8px 11px -5px rgba(0,0,0,0.2),0px 17px 26px 2px rgba(0,0,0,0.14),0px 6px 32px 5px rgba(0,0,0,0.12)", - "0px 9px 11px -5px rgba(0,0,0,0.2),0px 18px 28px 2px rgba(0,0,0,0.14),0px 7px 34px 6px rgba(0,0,0,0.12)", - "0px 9px 12px -6px rgba(0,0,0,0.2),0px 19px 29px 2px rgba(0,0,0,0.14),0px 7px 36px 6px rgba(0,0,0,0.12)", - "0px 10px 13px -6px rgba(0,0,0,0.2),0px 20px 31px 3px rgba(0,0,0,0.14),0px 8px 38px 7px rgba(0,0,0,0.12)", - "0px 10px 13px -6px rgba(0,0,0,0.2),0px 21px 33px 3px rgba(0,0,0,0.14),0px 8px 40px 7px rgba(0,0,0,0.12)", - "0px 10px 14px -6px rgba(0,0,0,0.2),0px 22px 35px 3px rgba(0,0,0,0.14),0px 8px 42px 7px rgba(0,0,0,0.12)", - "0px 11px 14px -7px rgba(0,0,0,0.2),0px 23px 36px 3px rgba(0,0,0,0.14),0px 9px 44px 8px rgba(0,0,0,0.12)", - "0px 11px 15px -7px rgba(0,0,0,0.2),0px 24px 38px 3px rgba(0,0,0,0.14),0px 9px 46px 8px rgba(0,0,0,0.12)" - ], - "typography": { - "htmlFontSize": 16, - "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", - "fontSize": 14, - "fontWeightLight": 300, - "fontWeightRegular": 400, - "fontWeightMedium": 500, - "fontWeightBold": 600, - "h1": { - "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", - "fontWeight": 300, - "fontSize": "6rem", - "lineHeight": 1.167, - "letterSpacing": "-0.01562em" - }, - "h2": { - "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", - "fontWeight": 300, - "fontSize": "3.75rem", - "lineHeight": 1.2, - "letterSpacing": "-0.00833em" - }, - "h3": { - "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", - "fontWeight": 400, - "fontSize": "3rem", - "lineHeight": 1.167, - "letterSpacing": "0em" - }, - "h4": { - "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", - "fontWeight": 400, - "fontSize": "2.125rem", - "lineHeight": 1.235, - "letterSpacing": "0.00735em" - }, - "h5": { - "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", - "fontWeight": 400, - "fontSize": "1.2rem", - "lineHeight": 1.334, - "letterSpacing": "0.02em" - }, - "h6": { - "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", - "fontWeight": 400, - "fontSize": "1rem", - "lineHeight": 1.6, - "letterSpacing": "0.0075em", - "textTransform": "uppercase" - }, - "subtitle1": { - "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", - "fontWeight": 400, - "fontSize": "1rem", - "lineHeight": 1.75, - "letterSpacing": "0.00938em" - }, - "subtitle2": { - "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", - "fontWeight": 500, - "fontSize": "0.875rem", - "lineHeight": 1.57, - "letterSpacing": "0.00714em" - }, - "body1": { - "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", - "fontWeight": 400, - "fontSize": "1rem", - "lineHeight": 1.5, - "letterSpacing": "0.00938em" - }, - "body2": { - "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", - "fontWeight": 400, - "fontSize": "0.875rem", - "lineHeight": 1.43, - "letterSpacing": "0.01071em" - }, - "button": { - "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", - "fontWeight": 500, - "fontSize": "0.875rem", - "lineHeight": 1.75, - "letterSpacing": "0.02857em", - "textTransform": "uppercase" - }, - "caption": { - "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", - "fontWeight": 400, - "fontSize": "0.8rem", - "lineHeight": 1.66, - "letterSpacing": "0.03333em" - }, - "overline": { - "fontFamily": "'Source Sans Pro', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif", - "fontWeight": 400, - "fontSize": "0.75rem", - "lineHeight": 2.66, - "letterSpacing": "0.08333em", - "textTransform": "uppercase" - } - }, - "shape": { - "borderRadius": 16 - }, - "transitions": { - "easing": { - "easeInOut": "cubic-bezier(0.4, 0, 0.2, 1)", - "easeOut": "cubic-bezier(0.0, 0, 0.2, 1)", - "easeIn": "cubic-bezier(0.4, 0, 1, 1)", - "sharp": "cubic-bezier(0.4, 0, 0.6, 1)" - }, - "duration": { - "shortest": 150, - "shorter": 200, - "short": 250, - "standard": 300, - "complex": 375, - "enteringScreen": 225, - "leavingScreen": 195 - } - }, - "zIndex": { - "mobileStepper": 1000, - "speedDial": 1050, - "appBar": 1100, - "drawer": 1200, - "modal": 1300, - "snackbar": 1400, - "tooltip": 1500 - } -} as IThemeOptions) as ITheme; - -theme.overrides = { - - ...theme.overrides, - - MuiTooltip: { - tooltip: { - backgroundColor: "#2a335a", - fontSize: "14px", - borderRadius: "8px", - } - }, - - "MuiButton": { - label: { - fontWeight: "bold", - }, - "contained": { - color: '#F7F9FC', - padding: "7px 18px", - }, - "containedPrimary": { - - color: '#F7F9FC', - - "&:disabled": { - backgroundColor: "rgba(0,0,0,0.15)", - color: "rgba(255, 255, 255, 0.8)" - } - }, - "outlinedPrimary": { - - padding: "8px 18px", - "&:hover": { - backgroundColor: "rgba(255, 255, 255, 0.06)" - }, - - "&:disabled": { - backgroundColor: "rgba(255, 255, 255, 0.06)", - color: "rgba(255, 255, 255, 0.25)" - }, - - } - }, - - "MuiSwitch": { - root: { - padding: 11, - width: 44, - height: 29, - }, - switchBase: { - '&$checked': { - transform: "translateX(10px)", - "& $thumb": { - backgroundColor: theme.palette.common.white, - }, - "&.MuiSwitch-colorPrimary + $track": { - backgroundColor: theme.palette.secondaryBackground.dark, - border: "3px solid " + theme.palette.primary.light, - opacity: 1, - }, - } - }, - thumb: { - width: 8, - height: 8, - border: "3px solid " + theme.palette.primary.main, - backgroundColor: theme.palette.primary.dark, - boxShadow: 'none', - }, - track: { - width: 14, - height: 4, - backgroundColor: theme.palette.secondaryBackground.dark, - border: "3px solid " + theme.palette.primary.light, - opacity: 1, - } - } -}; - -export default theme; - From 93714ab902efdbb11e68926c87b9b9f54bba30fd Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Mon, 5 Jul 2021 13:45:58 +0300 Subject: [PATCH 08/12] Update entries_controller.go --- api/pkg/controllers/entries_controller.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index 89dd450c2..f8645c38a 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -10,6 +10,7 @@ import ( "mizuserver/pkg/up9" "mizuserver/pkg/utils" "mizuserver/pkg/validation" + "strings" "time" ) @@ -88,9 +89,17 @@ func GetHARs(c *fiber.Ctx) error { for _, entryData := range entries { var harEntry har.Entry _ = json.Unmarshal([]byte(entryData.Entry), &harEntry) + if entryData.ResolvedDestination != "" { + harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entryData.ResolvedDestination) + } sourceOfEntry := entryData.ResolvedSource - fileName := fmt.Sprintf("%s.har", sourceOfEntry) + if sourceOfEntry != "" { + // naively assumes the proper service source is http + sourceOfEntry = fmt.Sprintf("http://%s", sourceOfEntry) + } + //replace / from the file name cause they end up creating a corrupted folder + fileName := fmt.Sprintf("%s.har", strings.ReplaceAll(sourceOfEntry, "/", "_")) if harOfSource, ok := harsObject[fileName]; ok { harOfSource.Log.Entries = append(harOfSource.Log.Entries, &harEntry) } else { @@ -104,11 +113,14 @@ func GetHARs(c *fiber.Ctx) error { Name: "mizu", Version: "0.0.2", }, - Source: sourceOfEntry, }, Entries: entriesHar, }, } + // leave undefined when no source is present, otherwise modeler assumes source is empty string "" + if sourceOfEntry != "" { + harsObject[fileName].Log.Creator.Source = sourceOfEntry + } } } From eef58496b5833a34e49ea1bfaf13abf9f5cd0626 Mon Sep 17 00:00:00 2001 From: gadotroee <55343099+gadotroee@users.noreply.github.com> Date: Mon, 5 Jul 2021 17:12:48 +0300 Subject: [PATCH 09/12] Add source and destination info when fetching entries (x-mizu-fields) (#93) --- api/pkg/database/main.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/pkg/database/main.go b/api/pkg/database/main.go index 4ce4f07db..f14734f85 100644 --- a/api/pkg/database/main.go +++ b/api/pkg/database/main.go @@ -63,6 +63,14 @@ func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []har.Entry { for _, entryData := range entries { var harEntry har.Entry _ = json.Unmarshal([]byte(entryData.Entry), &harEntry) + + if entryData.ResolvedSource != "" { + harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: entryData.ResolvedSource}) + } + if entryData.ResolvedDestination != "" { + harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entryData.ResolvedDestination}) + } + entriesArray = append(entriesArray, harEntry) } return entriesArray From 0f52533cd8b7d75f06dd9ab9c47e0d0e2f6bc592 Mon Sep 17 00:00:00 2001 From: Alon Girmonsky <1990761+alongir@users.noreply.github.com> Date: Tue, 6 Jul 2021 13:21:58 +0300 Subject: [PATCH 10/12] Update README.md (#82) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1d0cd48ef..9e989232a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # 水 mizu -standalone web app traffic viewer for Kubernetes +A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined. ## Download @@ -29,7 +29,7 @@ Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page. ## How to run 1. Find pod you'd like to tap to in your Kubernetes cluster -2. Run `mizu PODNAME` or `mizu REGEX` +2. Run `mizu tap PODNAME` or `mizu tap REGEX` 3. Open browser on `http://localhost:8899` as instructed .. 4. Watch the WebAPI traffic flowing .. 5. Type ^C to stop From 566eab3527c3998a6afd3c759532ef3402fbf7f5 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Tue, 6 Jul 2021 13:47:15 +0300 Subject: [PATCH 11/12] Update entries_controller.go and models.go --- api/pkg/controllers/entries_controller.go | 9 ++++++--- api/pkg/models/models.go | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index f8645c38a..a2139b297 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -93,13 +93,16 @@ func GetHARs(c *fiber.Ctx) error { harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entryData.ResolvedDestination) } + var fileName string sourceOfEntry := entryData.ResolvedSource if sourceOfEntry != "" { // naively assumes the proper service source is http sourceOfEntry = fmt.Sprintf("http://%s", sourceOfEntry) + //replace / from the file name cause they end up creating a corrupted folder + fileName = fmt.Sprintf("%s.har", strings.ReplaceAll(sourceOfEntry, "/", "_")) + } else { + fileName = "unknown_source.har" } - //replace / from the file name cause they end up creating a corrupted folder - fileName := fmt.Sprintf("%s.har", strings.ReplaceAll(sourceOfEntry, "/", "_")) if harOfSource, ok := harsObject[fileName]; ok { harOfSource.Log.Entries = append(harOfSource.Log.Entries, &harEntry) } else { @@ -119,7 +122,7 @@ func GetHARs(c *fiber.Ctx) error { } // leave undefined when no source is present, otherwise modeler assumes source is empty string "" if sourceOfEntry != "" { - harsObject[fileName].Log.Creator.Source = sourceOfEntry + harsObject[fileName].Log.Creator.Source = &sourceOfEntry } } } diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index a45124255..232a5c1f8 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -105,5 +105,5 @@ type ExtendedLog struct { type ExtendedCreator struct { *har.Creator - Source string `json:"_source"` + Source *string `json:"_source"` } From 12d873d34430f52beb008d5dbcbd65b77a787979 Mon Sep 17 00:00:00 2001 From: nimrod-up9 <59927337+nimrod-up9@users.noreply.github.com> Date: Tue, 6 Jul 2021 16:08:27 +0300 Subject: [PATCH 12/12] Fix Mizu sometimes resolves without namespace (#96) * Reordered imports. * Pass all ConnectionInfo to saveHarToDb. * Resolve destination by IP:Port instead of host name. --- api/pkg/api/main.go | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/api/pkg/api/main.go b/api/pkg/api/main.go index 532344f3e..4df93f500 100644 --- a/api/pkg/api/main.go +++ b/api/pkg/api/main.go @@ -5,10 +5,6 @@ import ( "context" "encoding/json" "fmt" - "mizuserver/pkg/database" - "mizuserver/pkg/models" - "mizuserver/pkg/resolver" - "mizuserver/pkg/utils" "net/url" "os" "path" @@ -19,6 +15,11 @@ import ( "github.com/google/martian/har" "github.com/up9inc/mizu/tap" "go.mongodb.org/mongo-driver/bson/primitive" + + "mizuserver/pkg/database" + "mizuserver/pkg/models" + "mizuserver/pkg/resolver" + "mizuserver/pkg/utils" ) var k8sResolver *resolver.Resolver @@ -84,7 +85,14 @@ func startReadingFiles(workingDir string) { for _, entry := range inputHar.Log.Entries { time.Sleep(time.Millisecond * 250) - saveHarToDb(entry, fileInfo.Name(), false) + connectionInfo := &tap.ConnectionInfo{ + ClientIP: fileInfo.Name(), + ClientPort: "", + ServerIP: "", + ServerPort: "", + IsOutgoing: false, + } + saveHarToDb(entry, connectionInfo) } rmErr := os.Remove(inputFilePath) utils.CheckErr(rmErr) @@ -97,7 +105,7 @@ func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) { } for item := range outputItems { - saveHarToDb(item.HarEntry, item.ConnectionInfo.ClientIP, item.ConnectionInfo.IsOutgoing) + saveHarToDb(item.HarEntry, item.ConnectionInfo) } } @@ -109,17 +117,17 @@ func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) { } -func saveHarToDb(entry *har.Entry, sender string, isOutgoing bool) { +func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) { entryBytes, _ := json.Marshal(entry) - serviceName, urlPath, serviceHostName := getServiceNameFromUrl(entry.Request.URL) + serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL) entryId := primitive.NewObjectID().Hex() var ( resolvedSource string resolvedDestination string ) if k8sResolver != nil { - resolvedSource = k8sResolver.Resolve(sender) - resolvedDestination = k8sResolver.Resolve(serviceHostName) + resolvedSource = k8sResolver.Resolve(connectionInfo.ClientIP) + resolvedDestination = k8sResolver.Resolve(fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort)) } mizuEntry := models.MizuEntry{ EntryId: entryId, @@ -129,11 +137,11 @@ func saveHarToDb(entry *har.Entry, sender string, isOutgoing bool) { Path: urlPath, Method: entry.Request.Method, Status: entry.Response.Status, - RequestSenderIp: sender, + RequestSenderIp: connectionInfo.ClientIP, Timestamp: entry.StartedDateTime.UnixNano() / int64(time.Millisecond), ResolvedSource: resolvedSource, ResolvedDestination: resolvedDestination, - IsOutgoing: isOutgoing, + IsOutgoing: connectionInfo.IsOutgoing, } database.GetEntriesTable().Create(&mizuEntry) @@ -142,10 +150,10 @@ func saveHarToDb(entry *har.Entry, sender string, isOutgoing bool) { broadcastToBrowserClients(baseEntryBytes) } -func getServiceNameFromUrl(inputUrl string) (string, string, string) { +func getServiceNameFromUrl(inputUrl string) (string, string) { parsed, err := url.Parse(inputUrl) utils.CheckErr(err) - return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host), parsed.Path, parsed.Host + return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host), parsed.Path } func CheckIsServiceIP(address string) bool {