From 79816ae3377d42a71cf46451017978799da07b14 Mon Sep 17 00:00:00 2001 From: Igor Gov Date: Tue, 13 Jul 2021 17:26:22 +0300 Subject: [PATCH 1/2] Adding the upload interval as parameter to tap function --- api/pkg/controllers/entries_controller.go | 2 +- api/pkg/models/models.go | 5 ++--- api/pkg/up9/main.go | 4 ++-- cli/cmd/tap.go | 2 ++ cli/cmd/tapRunner.go | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index eb4c6109c..cde3dda1d 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -159,7 +159,7 @@ func UploadEntries(c *fiber.Ctx) error { return c.Status(fiber.StatusServiceUnavailable).SendString("Can't get token") } rlog.Infof("Upload entries - uploading. token: %s model: %s\n", token.Token, token.Model) - go up9.UploadEntriesImpl(token.Token, token.Model, uploadRequestBody.Dest) + go up9.UploadEntriesImpl(token.Token, token.Model, uploadRequestBody.Dest, uploadRequestBody.sleepIntervalSec) return c.Status(fiber.StatusOK).SendString("OK") } diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index 8c0d07e22..f0be0cbc7 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -100,8 +100,6 @@ func (fedex *FullEntryDetailsExtra) UnmarshalData(entry *MizuEntry) error { return nil } - - type EntryData struct { Entry string `json:"entry,omitempty"` ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` @@ -114,7 +112,8 @@ type EntriesFilter struct { } type UploadEntriesRequestBody struct { - Dest string `query:"dest"` + Dest string `query:"dest"` + sleepIntervalSec int `query:"interval"` } type HarFetchRequestBody struct { diff --git a/api/pkg/up9/main.go b/api/pkg/up9/main.go index 9e2179edd..469b2eb99 100644 --- a/api/pkg/up9/main.go +++ b/api/pkg/up9/main.go @@ -112,13 +112,13 @@ func GetAnalyzeInfo() *shared.AnalyzeStatus { } } -func UploadEntriesImpl(token string, model string, envPrefix string) { +func UploadEntriesImpl(token string, model string, envPrefix string, sleepIntervalSec int) { analyzeInformation.IsAnalyzing = true analyzeInformation.AnalyzedModel = model analyzeInformation.AnalyzeToken = token analyzeInformation.AnalyzeDestination = envPrefix - sleepTime := time.Second * 10 + sleepTime := time.Second * time.Duration(sleepIntervalSec) var timestampFrom int64 = 0 diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 47f209738..e7b88b1be 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -21,6 +21,7 @@ type MizuTapOptions struct { MizuImage string PlainTextFilterRegexes []string TapOutgoing bool + SleepIntervalSec uint16 } var mizuTapOptions = &MizuTapOptions{} @@ -64,6 +65,7 @@ func init() { tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector") tapCmd.Flags().BoolVar(&mizuTapOptions.Analyze, "analyze", false, "Uploads traffic to UP9 for further analysis (Beta)") tapCmd.Flags().StringVar(&mizuTapOptions.AnalyzeDestination, "dest", "up9.app", "Destination environment") + tapCmd.Flags().Uint16VarP(&mizuTapOptions.SleepIntervalSec, "upload-interval", "", 10, "Interval in seconds for uploading data to UP9") tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces") tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file") tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector") diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 69a6f2015..43b5b2240 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -253,7 +253,7 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready if tappingOptions.Analyze { - url_path := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s", mizuProxiedUrl, url.QueryEscape(tappingOptions.AnalyzeDestination)) + url_path := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%s", mizuProxiedUrl, url.QueryEscape(tappingOptions.AnalyzeDestination), tappingOptions.SleepIntervalSec) u, err := url.ParseRequestURI(url_path) if err != nil { log.Fatal(fmt.Sprintf("Failed parsing the URL %v\n", err)) From 8fab07494cc173d3ee3a12f8cb340f342d1cfd99 Mon Sep 17 00:00:00 2001 From: Igor Gov Date: Tue, 13 Jul 2021 18:09:32 +0300 Subject: [PATCH 2/2] Adding the upload interval as parameter to tap function --- api/pkg/controllers/entries_controller.go | 6 +++--- api/pkg/models/models.go | 2 +- api/pkg/up9/main.go | 2 +- cli/cmd/tapRunner.go | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index cde3dda1d..af96feb18 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -141,7 +141,7 @@ func GetHARs(c *fiber.Ctx) error { } func UploadEntries(c *fiber.Ctx) error { - rlog.Debugf("Upload entries - started\n") + rlog.Infof("Upload entries - started\n") uploadRequestBody := &models.UploadEntriesRequestBody{} if err := c.QueryParser(uploadRequestBody); err != nil { @@ -153,13 +153,13 @@ func UploadEntries(c *fiber.Ctx) error { if up9.GetAnalyzeInfo().IsAnalyzing { return c.Status(fiber.StatusBadRequest).SendString("Cannot analyze, mizu is already analyzing") } - rlog.Debugf("Upload entries - creating token. dest %s\n", uploadRequestBody.Dest) + rlog.Infof("Upload entries - creating token. dest %s\n", uploadRequestBody.Dest) token, err := up9.CreateAnonymousToken(uploadRequestBody.Dest) if err != nil { return c.Status(fiber.StatusServiceUnavailable).SendString("Can't get token") } rlog.Infof("Upload entries - uploading. token: %s model: %s\n", token.Token, token.Model) - go up9.UploadEntriesImpl(token.Token, token.Model, uploadRequestBody.Dest, uploadRequestBody.sleepIntervalSec) + go up9.UploadEntriesImpl(token.Token, token.Model, uploadRequestBody.Dest, uploadRequestBody.SleepIntervalSec) return c.Status(fiber.StatusOK).SendString("OK") } diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index f0be0cbc7..534cc8def 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -113,7 +113,7 @@ type EntriesFilter struct { type UploadEntriesRequestBody struct { Dest string `query:"dest"` - sleepIntervalSec int `query:"interval"` + SleepIntervalSec int `query:"interval"` } type HarFetchRequestBody struct { diff --git a/api/pkg/up9/main.go b/api/pkg/up9/main.go index 469b2eb99..5b6d37810 100644 --- a/api/pkg/up9/main.go +++ b/api/pkg/up9/main.go @@ -36,7 +36,7 @@ func getGuestToken(url string, target *GuestToken) error { return err } defer resp.Body.Close() - rlog.Debugf("Got token from the server, starting to json decode... status code: %v", resp.StatusCode) + rlog.Infof("Got token from the server, starting to json decode... status code: %v", resp.StatusCode) return json.NewDecoder(resp.Body).Decode(target) } diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 43b5b2240..5ce3d5c4d 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -253,14 +253,14 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready if tappingOptions.Analyze { - url_path := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%s", mizuProxiedUrl, url.QueryEscape(tappingOptions.AnalyzeDestination), tappingOptions.SleepIntervalSec) + url_path := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(tappingOptions.AnalyzeDestination), tappingOptions.SleepIntervalSec) u, err := url.ParseRequestURI(url_path) if err != nil { log.Fatal(fmt.Sprintf("Failed parsing the URL %v\n", err)) } rlog.Debugf("Sending get request to %v\n", u.String()) - if response, err := http.Get(u.String()); err != nil && response.StatusCode != 200 { - fmt.Printf("error sending upload entries req %v\n", err) + if response, err := http.Get(u.String()); err != nil || response.StatusCode != 200 { + fmt.Printf("error sending upload entries req, status code: %v, err: %v\n", response.StatusCode, err) } else { fmt.Printf(mizu.Purple, "Traffic is uploading to UP9 for further analsys") fmt.Println()