diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 400bf3bdc..312db265d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -4,6 +4,10 @@ on: branches: - 'develop' - 'main' + push: + branches: + - 'develop' + - 'main' jobs: build: name: Build diff --git a/Makefile b/Makefile index a3515ff83..3d66f655f 100644 --- a/Makefile +++ b/Makefile @@ -67,3 +67,4 @@ clean-docker: test: ## Run tests. @echo "running cli tests"; cd cli && $(MAKE) test + @echo "running agent tests"; cd agent && $(MAKE) test diff --git a/agent/Makefile b/agent/Makefile new file mode 100644 index 000000000..c27ad9c63 --- /dev/null +++ b/agent/Makefile @@ -0,0 +1,2 @@ +test: ## Run agent tests. + @go test ./... -race -coverprofile=coverage.out -covermode=atomic diff --git a/agent/pkg/api/main.go b/agent/pkg/api/main.go index 942f09569..90a411ee2 100644 --- a/agent/pkg/api/main.go +++ b/agent/pkg/api/main.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "mizuserver/pkg/holder" + "mizuserver/pkg/providers" "net/url" "os" "path" @@ -108,6 +109,7 @@ func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) { } for item := range outputItems { + providers.EntryAdded() saveHarToDb(item.HarEntry, item.ConnectionInfo) } } diff --git a/agent/pkg/controllers/entries_controller.go b/agent/pkg/controllers/entries_controller.go index a1ad22d42..99bb8d8fb 100644 --- a/agent/pkg/controllers/entries_controller.go +++ b/agent/pkg/controllers/entries_controller.go @@ -241,14 +241,7 @@ func DeleteAllEntries(c *gin.Context) { } func GetGeneralStats(c *gin.Context) { - sqlQuery := "SELECT count(*) as count, min(timestamp) as min, max(timestamp) as max from mizu_entries" - var result struct { - Count int - Min int - Max int - } - database.GetEntriesTable().Raw(sqlQuery).Scan(&result) - c.JSON(http.StatusOK, result) + c.JSON(http.StatusOK, providers.GetGeneralStats()) } func GetTappingStatus(c *gin.Context) { diff --git a/agent/pkg/providers/stats_provider.go b/agent/pkg/providers/stats_provider.go new file mode 100644 index 000000000..0f42f0fb8 --- /dev/null +++ b/agent/pkg/providers/stats_provider.go @@ -0,0 +1,36 @@ +package providers + +import ( + "reflect" + "time" +) + +type GeneralStats struct { + EntriesCount int + FirstEntryTimestamp int + LastEntryTimestamp int +} + +var generalStats = GeneralStats{} + +func ResetGeneralStats() { + generalStats = GeneralStats{} +} + +func GetGeneralStats() GeneralStats { + return generalStats +} + +func EntryAdded() { + generalStats.EntriesCount++ + + currentTimestamp := int(time.Now().Unix()) + + if reflect.Value.IsZero(reflect.ValueOf(generalStats.FirstEntryTimestamp)) { + generalStats.FirstEntryTimestamp = currentTimestamp + } + + generalStats.LastEntryTimestamp = currentTimestamp +} + + diff --git a/agent/pkg/providers/stats_provider_test.go b/agent/pkg/providers/stats_provider_test.go new file mode 100644 index 000000000..a35f7ca68 --- /dev/null +++ b/agent/pkg/providers/stats_provider_test.go @@ -0,0 +1,35 @@ +package providers_test + +import ( + "fmt" + "mizuserver/pkg/providers" + "testing" +) + +func TestNoEntryAddedCount(t *testing.T) { + entriesStats := providers.GetGeneralStats() + + if entriesStats.EntriesCount != 0 { + t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesCount) + } +} + +func TestEntryAddedCount(t *testing.T) { + tests := []int{1, 5, 10, 100, 500, 1000} + + for _, entriesCount := range tests { + t.Run(fmt.Sprintf("EntriesCount%v", entriesCount), func(t *testing.T) { + t.Cleanup(providers.ResetGeneralStats) + + for i := 0; i < entriesCount; i++ { + providers.EntryAdded() + } + + entriesStats := providers.GetGeneralStats() + + if entriesStats.EntriesCount != entriesCount { + t.Errorf("unexpected result - expected: %v, actual: %v", entriesCount, entriesStats.EntriesCount) + } + }) + } +} diff --git a/cli/cmd/config.go b/cli/cmd/config.go index 6fc666fb5..0a6dab3ea 100644 --- a/cli/cmd/config.go +++ b/cli/cmd/config.go @@ -5,6 +5,7 @@ import ( "github.com/spf13/cobra" "github.com/up9inc/mizu/cli/config" "github.com/up9inc/mizu/cli/logger" + "github.com/up9inc/mizu/cli/telemetry" "github.com/up9inc/mizu/cli/uiUtils" "io/ioutil" ) @@ -15,6 +16,8 @@ var configCmd = &cobra.Command{ Use: "config", Short: "Generate config with default values", RunE: func(cmd *cobra.Command, args []string) error { + go telemetry.ReportRun("config", config.Config) + template, err := config.GetConfigWithDefaults() if err != nil { logger.Log.Errorf("Failed generating config with defaults %v", err) diff --git a/cli/cmd/fetch.go b/cli/cmd/fetch.go index f26f43c44..f18e5bf3f 100644 --- a/cli/cmd/fetch.go +++ b/cli/cmd/fetch.go @@ -14,6 +14,7 @@ var fetchCmd = &cobra.Command{ Short: "Download recorded traffic to files", RunE: func(cmd *cobra.Command, args []string) error { go telemetry.ReportRun("fetch", config.Config.Fetch) + if isCompatible, err := version.CheckVersionCompatibility(config.Config.Fetch.GuiPort); err != nil { return err } else if !isCompatible { diff --git a/cli/cmd/logs.go b/cli/cmd/logs.go index 8f7a2f42e..838fc8dbb 100644 --- a/cli/cmd/logs.go +++ b/cli/cmd/logs.go @@ -7,6 +7,7 @@ import ( "github.com/up9inc/mizu/cli/kubernetes" "github.com/up9inc/mizu/cli/logger" "github.com/up9inc/mizu/cli/mizu/fsUtils" + "github.com/up9inc/mizu/cli/telemetry" "os" "path" ) @@ -17,6 +18,8 @@ var logsCmd = &cobra.Command{ Use: "logs", Short: "Create a zip file with logs for Github issue or troubleshoot", RunE: func(cmd *cobra.Command, args []string) error { + go telemetry.ReportRun("logs", config.Config) + kubernetesProvider, err := kubernetes.NewProvider(config.Config.View.KubeConfigPath) if err != nil { return nil diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index be0aac841..6255bea18 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -11,6 +11,7 @@ import ( "github.com/up9inc/mizu/cli/mizu/fsUtils" "github.com/up9inc/mizu/cli/mizu/goUtils" "github.com/up9inc/mizu/cli/mizu/version" + "github.com/up9inc/mizu/cli/telemetry" "net/http" "net/url" "os" @@ -100,7 +101,7 @@ func RunMizuTap() { nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods) - defer cleanUpMizuResources(kubernetesProvider) + defer cleanUpMizu(kubernetesProvider) if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions, mizuValidationRules); err != nil { logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err))) return @@ -249,8 +250,12 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi return nil } -func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) { +func cleanUpMizu(kubernetesProvider *kubernetes.Provider) { + telemetry.ReportAPICalls(config.Config.Tap.GuiPort) + cleanUpMizuResources(kubernetesProvider) +} +func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) { removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) defer cancel() diff --git a/cli/cmd/version.go b/cli/cmd/version.go index 7d3f9029d..816f92865 100644 --- a/cli/cmd/version.go +++ b/cli/cmd/version.go @@ -18,6 +18,7 @@ var versionCmd = &cobra.Command{ Short: "Print version info", RunE: func(cmd *cobra.Command, args []string) error { go telemetry.ReportRun("version", config.Config.Version) + if config.Config.Version.DebugInfo { timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0) logger.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash) diff --git a/cli/telemetry/telemetry.go b/cli/telemetry/telemetry.go index 6dd438877..258d88dfa 100644 --- a/cli/telemetry/telemetry.go +++ b/cli/telemetry/telemetry.go @@ -5,39 +5,105 @@ import ( "encoding/json" "fmt" "github.com/up9inc/mizu/cli/config" + "github.com/up9inc/mizu/cli/kubernetes" "github.com/up9inc/mizu/cli/logger" "github.com/up9inc/mizu/cli/mizu" + "io/ioutil" "net/http" ) const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry" func ReportRun(cmd string, args interface{}) { - if !config.Config.Telemetry { - logger.Log.Debugf("not reporting due to config value") + if !shouldRunTelemetry() { + logger.Log.Debugf("not reporting telemetry") return } - if mizu.Branch != "main" && mizu.Branch != "develop" { - logger.Log.Debugf("not reporting telemetry on private branches") + argsBytes, _ := json.Marshal(args) + argsMap := map[string]interface{}{ + "cmd": cmd, + "args": string(argsBytes), } - argsBytes, _ := json.Marshal(args) - argsMap := map[string]string{ - "telemetry_type": "execution", - "cmd": cmd, - "args": string(argsBytes), - "component": "mizu_cli", - "BuildTimestamp": mizu.BuildTimestamp, - "Branch": mizu.Branch, - "version": mizu.SemVer} - argsMap["message"] = fmt.Sprintf("mizu %v - %v", argsMap["cmd"], string(argsBytes)) + if err := sendTelemetry("Execution", argsMap); err != nil { + logger.Log.Debug(err) + return + } + + logger.Log.Debugf("successfully reported telemetry for cmd %v", cmd) +} + +func ReportAPICalls(mizuPort uint16) { + if !shouldRunTelemetry() { + logger.Log.Debugf("not reporting telemetry") + return + } + + mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizuPort) + generalStatsUrl := fmt.Sprintf("http://%s/api/generalStats", mizuProxiedUrl) + + response, requestErr := http.Get(generalStatsUrl) + if requestErr != nil { + logger.Log.Debugf("ERROR: failed to get general stats for telemetry, err: %v", requestErr) + return + } else if response.StatusCode != 200 { + logger.Log.Debugf("ERROR: failed to get general stats for telemetry, status code: %v", response.StatusCode) + return + } + + defer func() { _ = response.Body.Close() }() + + data, readErr := ioutil.ReadAll(response.Body) + if readErr != nil { + logger.Log.Debugf("ERROR: failed to read general stats for telemetry, err: %v", readErr) + return + } + + var generalStats map[string]interface{} + if parseErr := json.Unmarshal(data, &generalStats); parseErr != nil { + logger.Log.Debugf("ERROR: failed to parse general stats for telemetry, err: %v", parseErr) + return + } + + argsMap := map[string]interface{}{ + "apiCallsCount": generalStats["EntriesCount"], + "firstAPICallTimestamp": generalStats["FirstEntryTimestamp"], + "lastAPICallTimestamp": generalStats["LastEntryTimestamp"], + } + + if err := sendTelemetry("APICalls", argsMap); err != nil { + logger.Log.Debug(err) + return + } + + logger.Log.Debugf("successfully reported telemetry of api calls") +} + +func shouldRunTelemetry() bool { + if !config.Config.Telemetry { + return false + } + + if mizu.Branch != "main" && mizu.Branch != "develop" { + return false + } + + return true +} + +func sendTelemetry(telemetryType string, argsMap map[string]interface{}) error { + argsMap["telemetryType"] = telemetryType + argsMap["component"] = "mizu_cli" + argsMap["buildTimestamp"] = mizu.BuildTimestamp + argsMap["branch"] = mizu.Branch + argsMap["version"] = mizu.SemVer jsonValue, _ := json.Marshal(argsMap) if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil { - logger.Log.Debugf("error sending telemetry err: %v, response %v", err, resp) - } else { - logger.Log.Debugf("Successfully reported telemetry") + return fmt.Errorf("ERROR: failed sending telemetry, err: %v, response %v", err, resp) } + + return nil }