added telemetry api calls (#201)

.github/workflows/test.yaml

@@ -4,6 +4,10 @@ on:
     branches:
       - 'develop'
       - 'main'
+  push:
+    branches:
+      - 'develop'
+      - 'main'
 jobs:
   build:
     name: Build

Makefile

@@ -67,3 +67,4 @@ clean-docker:
 
 test: ## Run tests.
 	@echo "running cli tests"; cd cli && $(MAKE) test
+	@echo "running agent tests"; cd agent && $(MAKE) test

agent/Makefile

@@ -0,0 +1,2 @@
+test: ## Run agent tests.
+	@go test ./... -race -coverprofile=coverage.out -covermode=atomic

agent/pkg/api/main.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"mizuserver/pkg/holder"
+	"mizuserver/pkg/providers"
 	"net/url"
 	"os"
 	"path"
@@ -108,6 +109,7 @@ func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) {
 	}
 
 	for item := range outputItems {
+		providers.EntryAdded()
 		saveHarToDb(item.HarEntry, item.ConnectionInfo)
 	}
 }

agent/pkg/controllers/entries_controller.go

@@ -241,14 +241,7 @@ func DeleteAllEntries(c *gin.Context) {
 }
 
 func GetGeneralStats(c *gin.Context) {
-	sqlQuery := "SELECT count(*) as count, min(timestamp) as min, max(timestamp) as max from mizu_entries"
-	var result struct {
-		Count int
-		Min   int
-		Max   int
-	}
-	database.GetEntriesTable().Raw(sqlQuery).Scan(&result)
-	c.JSON(http.StatusOK, result)
+	c.JSON(http.StatusOK, providers.GetGeneralStats())
 }
 
 func GetTappingStatus(c *gin.Context) {

agent/pkg/providers/stats_provider.go

@@ -0,0 +1,36 @@
+package providers
+
+import (
+	"reflect"
+	"time"
+)
+
+type GeneralStats struct {
+	EntriesCount        int
+	FirstEntryTimestamp int
+	LastEntryTimestamp  int
+}
+
+var generalStats = GeneralStats{}
+
+func ResetGeneralStats() {
+	generalStats = GeneralStats{}
+}
+
+func GetGeneralStats() GeneralStats {
+	return generalStats
+}
+
+func EntryAdded() {
+	generalStats.EntriesCount++
+
+	currentTimestamp := int(time.Now().Unix())
+
+	if reflect.Value.IsZero(reflect.ValueOf(generalStats.FirstEntryTimestamp)) {
+		generalStats.FirstEntryTimestamp = currentTimestamp
+	}
+
+	generalStats.LastEntryTimestamp = currentTimestamp
+}
+
+

agent/pkg/providers/stats_provider_test.go

@@ -0,0 +1,35 @@
+package providers_test
+
+import (
+	"fmt"
+	"mizuserver/pkg/providers"
+	"testing"
+)
+
+func TestNoEntryAddedCount(t *testing.T) {
+	entriesStats := providers.GetGeneralStats()
+
+	if entriesStats.EntriesCount != 0 {
+		t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesCount)
+	}
+}
+
+func TestEntryAddedCount(t *testing.T) {
+	tests := []int{1, 5, 10, 100, 500, 1000}
+
+	for _, entriesCount := range tests {
+		t.Run(fmt.Sprintf("EntriesCount%v", entriesCount), func(t *testing.T) {
+			t.Cleanup(providers.ResetGeneralStats)
+
+			for i := 0; i < entriesCount; i++ {
+				providers.EntryAdded()
+			}
+
+			entriesStats := providers.GetGeneralStats()
+
+			if entriesStats.EntriesCount != entriesCount {
+				t.Errorf("unexpected result - expected: %v, actual: %v", entriesCount, entriesStats.EntriesCount)
+			}
+		})
+	}
+}

cli/cmd/config.go

@@ -5,6 +5,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"io/ioutil"
 )
@@ -15,6 +16,8 @@ var configCmd = &cobra.Command{
 	Use:   "config",
 	Short: "Generate config with default values",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("config", config.Config)
+
 		template, err := config.GetConfigWithDefaults()
 		if err != nil {
 			logger.Log.Errorf("Failed generating config with defaults %v", err)

cli/cmd/fetch.go

@@ -14,6 +14,7 @@ var fetchCmd = &cobra.Command{
 	Short: "Download recorded traffic to files",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		go telemetry.ReportRun("fetch", config.Config.Fetch)
+
 		if isCompatible, err := version.CheckVersionCompatibility(config.Config.Fetch.GuiPort); err != nil {
 			return err
 		} else if !isCompatible {

cli/cmd/logs.go

@@ -7,6 +7,7 @@ import (
 	"github.com/up9inc/mizu/cli/kubernetes"
 	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"os"
 	"path"
 )
@@ -17,6 +18,8 @@ var logsCmd = &cobra.Command{
 	Use:   "logs",
 	Short: "Create a zip file with logs for Github issue or troubleshoot",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("logs", config.Config)
+
 		kubernetesProvider, err := kubernetes.NewProvider(config.Config.View.KubeConfigPath)
 		if err != nil {
 			return nil

cli/cmd/tapRunner.go

@@ -11,6 +11,7 @@ import (
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/mizu/goUtils"
 	"github.com/up9inc/mizu/cli/mizu/version"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"net/http"
 	"net/url"
 	"os"
@@ -100,7 +101,7 @@ func RunMizuTap() {
 
 	nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
 
-	defer cleanUpMizuResources(kubernetesProvider)
+	defer cleanUpMizu(kubernetesProvider)
 	if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions, mizuValidationRules); err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
 		return
@@ -249,8 +250,12 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
 	return nil
 }
 
-func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
+func cleanUpMizu(kubernetesProvider *kubernetes.Provider) {
+	telemetry.ReportAPICalls(config.Config.Tap.GuiPort)
+	cleanUpMizuResources(kubernetesProvider)
+}
 
+func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
 	removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
 	defer cancel()
 

cli/cmd/version.go

@@ -18,6 +18,7 @@ var versionCmd = &cobra.Command{
 	Short: "Print version info",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		go telemetry.ReportRun("version", config.Config.Version)
+
 		if config.Config.Version.DebugInfo {
 			timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
 			logger.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)

cli/telemetry/telemetry.go

@@ -5,39 +5,105 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/kubernetes"
 	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu"
+	"io/ioutil"
 	"net/http"
 )
 
 const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
 
 func ReportRun(cmd string, args interface{}) {
-	if !config.Config.Telemetry {
-		logger.Log.Debugf("not reporting due to config value")
+	if !shouldRunTelemetry() {
+		logger.Log.Debugf("not reporting telemetry")
 		return
 	}
 
-	if mizu.Branch != "main" && mizu.Branch != "develop" {
-		logger.Log.Debugf("not reporting telemetry on private branches")
+	argsBytes, _ := json.Marshal(args)
+	argsMap := map[string]interface{}{
+		"cmd":  cmd,
+		"args": string(argsBytes),
 	}
 
-	argsBytes, _ := json.Marshal(args)
-	argsMap := map[string]string{
-		"telemetry_type": "execution",
-		"cmd":            cmd,
-		"args":           string(argsBytes),
-		"component":      "mizu_cli",
-		"BuildTimestamp": mizu.BuildTimestamp,
-		"Branch":         mizu.Branch,
-		"version":        mizu.SemVer}
-	argsMap["message"] = fmt.Sprintf("mizu %v - %v", argsMap["cmd"], string(argsBytes))
+	if err := sendTelemetry("Execution", argsMap); err != nil {
+		logger.Log.Debug(err)
+		return
+	}
+
+	logger.Log.Debugf("successfully reported telemetry for cmd %v", cmd)
+}
+
+func ReportAPICalls(mizuPort uint16) {
+	if !shouldRunTelemetry() {
+		logger.Log.Debugf("not reporting telemetry")
+		return
+	}
+
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizuPort)
+	generalStatsUrl := fmt.Sprintf("http://%s/api/generalStats", mizuProxiedUrl)
+
+	response, requestErr := http.Get(generalStatsUrl)
+	if requestErr != nil {
+		logger.Log.Debugf("ERROR: failed to get general stats for telemetry, err: %v", requestErr)
+		return
+	} else if response.StatusCode != 200 {
+		logger.Log.Debugf("ERROR: failed to get general stats for telemetry, status code: %v", response.StatusCode)
+		return
+	}
+
+	defer func() { _ = response.Body.Close() }()
+
+	data, readErr := ioutil.ReadAll(response.Body)
+	if readErr != nil {
+		logger.Log.Debugf("ERROR: failed to read general stats for telemetry, err: %v", readErr)
+		return
+	}
+
+	var generalStats map[string]interface{}
+	if parseErr := json.Unmarshal(data, &generalStats); parseErr != nil {
+		logger.Log.Debugf("ERROR: failed to parse general stats for telemetry, err: %v", parseErr)
+		return
+	}
+
+	argsMap := map[string]interface{}{
+		"apiCallsCount":         generalStats["EntriesCount"],
+		"firstAPICallTimestamp": generalStats["FirstEntryTimestamp"],
+		"lastAPICallTimestamp":  generalStats["LastEntryTimestamp"],
+	}
+
+	if err := sendTelemetry("APICalls", argsMap); err != nil {
+		logger.Log.Debug(err)
+		return
+	}
+
+	logger.Log.Debugf("successfully reported telemetry of api calls")
+}
+
+func shouldRunTelemetry() bool {
+	if !config.Config.Telemetry {
+		return false
+	}
+
+	if mizu.Branch != "main" && mizu.Branch != "develop" {
+		return false
+	}
+
+	return true
+}
+
+func sendTelemetry(telemetryType string, argsMap map[string]interface{}) error {
+	argsMap["telemetryType"] = telemetryType
+	argsMap["component"] = "mizu_cli"
+	argsMap["buildTimestamp"] = mizu.BuildTimestamp
+	argsMap["branch"] = mizu.Branch
+	argsMap["version"] = mizu.SemVer
 
 	jsonValue, _ := json.Marshal(argsMap)
 
 	if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
-		logger.Log.Debugf("error sending telemetry err: %v, response %v", err, resp)
-	} else {
-		logger.Log.Debugf("Successfully reported telemetry")
+		return fmt.Errorf("ERROR: failed sending telemetry, err: %v, response %v", err, resp)
 	}
+
+	return nil
 }