added telemetry api calls (#201)

This commit is contained in:
RoyUP9
2021-08-11 15:57:41 +03:00
committed by GitHub
parent 8a8cf4aa77
commit c8e5886a96
13 changed files with 179 additions and 27 deletions

View File

@@ -4,6 +4,10 @@ on:
branches:
- 'develop'
- 'main'
push:
branches:
- 'develop'
- 'main'
jobs:
build:
name: Build

View File

@@ -67,3 +67,4 @@ clean-docker:
test: ## Run tests.
@echo "running cli tests"; cd cli && $(MAKE) test
@echo "running agent tests"; cd agent && $(MAKE) test

2
agent/Makefile Normal file
View File

@@ -0,0 +1,2 @@
test: ## Run agent tests.
@go test ./... -race -coverprofile=coverage.out -covermode=atomic

View File

@@ -6,6 +6,7 @@ import (
"encoding/json"
"fmt"
"mizuserver/pkg/holder"
"mizuserver/pkg/providers"
"net/url"
"os"
"path"
@@ -108,6 +109,7 @@ func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) {
}
for item := range outputItems {
providers.EntryAdded()
saveHarToDb(item.HarEntry, item.ConnectionInfo)
}
}

View File

@@ -241,14 +241,7 @@ func DeleteAllEntries(c *gin.Context) {
}
func GetGeneralStats(c *gin.Context) {
sqlQuery := "SELECT count(*) as count, min(timestamp) as min, max(timestamp) as max from mizu_entries"
var result struct {
Count int
Min int
Max int
}
database.GetEntriesTable().Raw(sqlQuery).Scan(&result)
c.JSON(http.StatusOK, result)
c.JSON(http.StatusOK, providers.GetGeneralStats())
}
func GetTappingStatus(c *gin.Context) {

View File

@@ -0,0 +1,36 @@
package providers
import (
"reflect"
"time"
)
type GeneralStats struct {
EntriesCount int
FirstEntryTimestamp int
LastEntryTimestamp int
}
var generalStats = GeneralStats{}
func ResetGeneralStats() {
generalStats = GeneralStats{}
}
func GetGeneralStats() GeneralStats {
return generalStats
}
func EntryAdded() {
generalStats.EntriesCount++
currentTimestamp := int(time.Now().Unix())
if reflect.Value.IsZero(reflect.ValueOf(generalStats.FirstEntryTimestamp)) {
generalStats.FirstEntryTimestamp = currentTimestamp
}
generalStats.LastEntryTimestamp = currentTimestamp
}

View File

@@ -0,0 +1,35 @@
package providers_test
import (
"fmt"
"mizuserver/pkg/providers"
"testing"
)
func TestNoEntryAddedCount(t *testing.T) {
entriesStats := providers.GetGeneralStats()
if entriesStats.EntriesCount != 0 {
t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesCount)
}
}
func TestEntryAddedCount(t *testing.T) {
tests := []int{1, 5, 10, 100, 500, 1000}
for _, entriesCount := range tests {
t.Run(fmt.Sprintf("EntriesCount%v", entriesCount), func(t *testing.T) {
t.Cleanup(providers.ResetGeneralStats)
for i := 0; i < entriesCount; i++ {
providers.EntryAdded()
}
entriesStats := providers.GetGeneralStats()
if entriesStats.EntriesCount != entriesCount {
t.Errorf("unexpected result - expected: %v, actual: %v", entriesCount, entriesStats.EntriesCount)
}
})
}
}

View File

@@ -5,6 +5,7 @@ import (
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/config"
"github.com/up9inc/mizu/cli/logger"
"github.com/up9inc/mizu/cli/telemetry"
"github.com/up9inc/mizu/cli/uiUtils"
"io/ioutil"
)
@@ -15,6 +16,8 @@ var configCmd = &cobra.Command{
Use: "config",
Short: "Generate config with default values",
RunE: func(cmd *cobra.Command, args []string) error {
go telemetry.ReportRun("config", config.Config)
template, err := config.GetConfigWithDefaults()
if err != nil {
logger.Log.Errorf("Failed generating config with defaults %v", err)

View File

@@ -14,6 +14,7 @@ var fetchCmd = &cobra.Command{
Short: "Download recorded traffic to files",
RunE: func(cmd *cobra.Command, args []string) error {
go telemetry.ReportRun("fetch", config.Config.Fetch)
if isCompatible, err := version.CheckVersionCompatibility(config.Config.Fetch.GuiPort); err != nil {
return err
} else if !isCompatible {

View File

@@ -7,6 +7,7 @@ import (
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/logger"
"github.com/up9inc/mizu/cli/mizu/fsUtils"
"github.com/up9inc/mizu/cli/telemetry"
"os"
"path"
)
@@ -17,6 +18,8 @@ var logsCmd = &cobra.Command{
Use: "logs",
Short: "Create a zip file with logs for Github issue or troubleshoot",
RunE: func(cmd *cobra.Command, args []string) error {
go telemetry.ReportRun("logs", config.Config)
kubernetesProvider, err := kubernetes.NewProvider(config.Config.View.KubeConfigPath)
if err != nil {
return nil

View File

@@ -11,6 +11,7 @@ import (
"github.com/up9inc/mizu/cli/mizu/fsUtils"
"github.com/up9inc/mizu/cli/mizu/goUtils"
"github.com/up9inc/mizu/cli/mizu/version"
"github.com/up9inc/mizu/cli/telemetry"
"net/http"
"net/url"
"os"
@@ -100,7 +101,7 @@ func RunMizuTap() {
nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
defer cleanUpMizuResources(kubernetesProvider)
defer cleanUpMizu(kubernetesProvider)
if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions, mizuValidationRules); err != nil {
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
return
@@ -249,8 +250,12 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
return nil
}
func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
func cleanUpMizu(kubernetesProvider *kubernetes.Provider) {
telemetry.ReportAPICalls(config.Config.Tap.GuiPort)
cleanUpMizuResources(kubernetesProvider)
}
func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
defer cancel()

View File

@@ -18,6 +18,7 @@ var versionCmd = &cobra.Command{
Short: "Print version info",
RunE: func(cmd *cobra.Command, args []string) error {
go telemetry.ReportRun("version", config.Config.Version)
if config.Config.Version.DebugInfo {
timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
logger.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)

View File

@@ -5,39 +5,105 @@ import (
"encoding/json"
"fmt"
"github.com/up9inc/mizu/cli/config"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/logger"
"github.com/up9inc/mizu/cli/mizu"
"io/ioutil"
"net/http"
)
const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
func ReportRun(cmd string, args interface{}) {
if !config.Config.Telemetry {
logger.Log.Debugf("not reporting due to config value")
if !shouldRunTelemetry() {
logger.Log.Debugf("not reporting telemetry")
return
}
if mizu.Branch != "main" && mizu.Branch != "develop" {
logger.Log.Debugf("not reporting telemetry on private branches")
argsBytes, _ := json.Marshal(args)
argsMap := map[string]interface{}{
"cmd": cmd,
"args": string(argsBytes),
}
argsBytes, _ := json.Marshal(args)
argsMap := map[string]string{
"telemetry_type": "execution",
"cmd": cmd,
"args": string(argsBytes),
"component": "mizu_cli",
"BuildTimestamp": mizu.BuildTimestamp,
"Branch": mizu.Branch,
"version": mizu.SemVer}
argsMap["message"] = fmt.Sprintf("mizu %v - %v", argsMap["cmd"], string(argsBytes))
if err := sendTelemetry("Execution", argsMap); err != nil {
logger.Log.Debug(err)
return
}
logger.Log.Debugf("successfully reported telemetry for cmd %v", cmd)
}
func ReportAPICalls(mizuPort uint16) {
if !shouldRunTelemetry() {
logger.Log.Debugf("not reporting telemetry")
return
}
mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizuPort)
generalStatsUrl := fmt.Sprintf("http://%s/api/generalStats", mizuProxiedUrl)
response, requestErr := http.Get(generalStatsUrl)
if requestErr != nil {
logger.Log.Debugf("ERROR: failed to get general stats for telemetry, err: %v", requestErr)
return
} else if response.StatusCode != 200 {
logger.Log.Debugf("ERROR: failed to get general stats for telemetry, status code: %v", response.StatusCode)
return
}
defer func() { _ = response.Body.Close() }()
data, readErr := ioutil.ReadAll(response.Body)
if readErr != nil {
logger.Log.Debugf("ERROR: failed to read general stats for telemetry, err: %v", readErr)
return
}
var generalStats map[string]interface{}
if parseErr := json.Unmarshal(data, &generalStats); parseErr != nil {
logger.Log.Debugf("ERROR: failed to parse general stats for telemetry, err: %v", parseErr)
return
}
argsMap := map[string]interface{}{
"apiCallsCount": generalStats["EntriesCount"],
"firstAPICallTimestamp": generalStats["FirstEntryTimestamp"],
"lastAPICallTimestamp": generalStats["LastEntryTimestamp"],
}
if err := sendTelemetry("APICalls", argsMap); err != nil {
logger.Log.Debug(err)
return
}
logger.Log.Debugf("successfully reported telemetry of api calls")
}
func shouldRunTelemetry() bool {
if !config.Config.Telemetry {
return false
}
if mizu.Branch != "main" && mizu.Branch != "develop" {
return false
}
return true
}
func sendTelemetry(telemetryType string, argsMap map[string]interface{}) error {
argsMap["telemetryType"] = telemetryType
argsMap["component"] = "mizu_cli"
argsMap["buildTimestamp"] = mizu.BuildTimestamp
argsMap["branch"] = mizu.Branch
argsMap["version"] = mizu.SemVer
jsonValue, _ := json.Marshal(argsMap)
if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
logger.Log.Debugf("error sending telemetry err: %v, response %v", err, resp)
} else {
logger.Log.Debugf("Successfully reported telemetry")
return fmt.Errorf("ERROR: failed sending telemetry, err: %v, response %v", err, resp)
}
return nil
}