From dba8b1f215ac975e9e3ae2f330aed73867c2fb57 Mon Sep 17 00:00:00 2001 From: Alon Girmonsky <1990761+alongir@users.noreply.github.com> Date: Fri, 20 Aug 2021 02:39:52 -0700 Subject: [PATCH 01/11] some changes in the read me (#241) change prerequisite to permissions and kubeconfig. These are more FYIs as Mizu requires very little prerequisites. Change the description to match getmizu.io --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6993e029b..4e84d10a3 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ # The API Traffic Viewer for Kubernetes -A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined +A simple-yet-powerful API traffic viewer for Kubernetes enabling you to view all API communication between microservices to help your debug and troubleshoot regressions. + +Think TCPDump and Chrome Dev Tools combined. ![Simple UI](assets/mizu-ui.png) @@ -38,8 +40,10 @@ SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/ ### Development (unstable) Build Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page -## Prerequisites -1. Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config` +## Kubeconfig & Permissions +While `mizu`most often works out of the box, you can influence its behavior: + +1. [OPTIONAL] Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config` 2. `mizu` assumes user running the command has permissions to create resources (such as pods, services, namespaces) on your Kubernetes cluster (no worries - `mizu` resources are cleaned up upon termination) For detailed list of k8s permissions see [PERMISSIONS](PERMISSIONS.md) document From afd5757315a65dc3ed9054889432555cade3a4b5 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Sun, 22 Aug 2021 11:38:19 +0300 Subject: [PATCH 02/11] added tapper count route and wait time for tappers in test (#226) --- acceptanceTests/tap_test.go | 125 +++++++++++---------- acceptanceTests/testsUtils.go | 78 ++++++++++--- agent/pkg/api/socket_server_handlers.go | 2 + agent/pkg/controllers/status_controller.go | 4 + agent/pkg/providers/status_provider.go | 18 ++- agent/pkg/routes/status_routes.go | 2 + 6 files changed, 150 insertions(+), 79 deletions(-) diff --git a/acceptanceTests/tap_test.go b/acceptanceTests/tap_test.go index 3c359a0d0..1e6ca9a4f 100644 --- a/acceptanceTests/tap_test.go +++ b/acceptanceTests/tap_test.go @@ -13,22 +13,22 @@ func TestTapAndFetch(t *testing.T) { t.Skip("ignored acceptance test") } - tests := []int{1, 100} + tests := []int{50} for _, entriesCount := range tests { t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) { - cliPath, cliPathErr := GetCliPath() + cliPath, cliPathErr := getCliPath() if cliPathErr != nil { t.Errorf("failed to get cli path, err: %v", cliPathErr) return } - tapCmdArgs := GetDefaultTapCommandArgs() + tapCmdArgs := getDefaultTapCommandArgs() tapCmd := exec.Command(cliPath, tapCmdArgs...) t.Logf("running command: %v", tapCmd.String()) t.Cleanup(func() { - if err := CleanupCommand(tapCmd); err != nil { + if err := cleanupCommand(tapCmd); err != nil { t.Logf("failed to cleanup tap command, err: %v", err) } }) @@ -38,86 +38,87 @@ func TestTapAndFetch(t *testing.T) { return } - time.Sleep(30 * time.Second) + if err := waitTapPodsReady(); err != nil { + t.Errorf("failed to start tap pods on time, err: %v", err) + return + } proxyUrl := "http://localhost:8080/api/v1/namespaces/mizu-tests/services/httpbin/proxy/get" for i := 0; i < entriesCount; i++ { - if _, requestErr := ExecuteHttpRequest(proxyUrl); requestErr != nil { + if _, requestErr := executeHttpRequest(proxyUrl); requestErr != nil { t.Errorf("failed to send proxy request, err: %v", requestErr) return } } - time.Sleep(5 * time.Second) - timestamp := time.Now().UnixNano() / int64(time.Millisecond) + entriesCheckFunc := func() error { + timestamp := time.Now().UnixNano() / int64(time.Millisecond) - entriesUrl := fmt.Sprintf("http://localhost:8899/mizu/api/entries?limit=%v&operator=lt×tamp=%v", entriesCount, timestamp) - requestResult, requestErr := ExecuteHttpRequest(entriesUrl) - if requestErr != nil { - t.Errorf("failed to get entries, err: %v", requestErr) + entriesUrl := fmt.Sprintf("http://localhost:8899/mizu/api/entries?limit=%v&operator=lt×tamp=%v", entriesCount, timestamp) + requestResult, requestErr := executeHttpRequest(entriesUrl) + if requestErr != nil { + return fmt.Errorf("failed to get entries, err: %v", requestErr) + } + + entries, ok := requestResult.([]interface{}) + if !ok { + return fmt.Errorf("invalid entries type") + } + + if len(entries) == 0 { + return fmt.Errorf("unexpected entries result - Expected more than 0 entries") + } + + entry, ok := entries[0].(map[string]interface{}) + if !ok { + return fmt.Errorf("invalid entry type") + } + + entryUrl := fmt.Sprintf("http://localhost:8899/mizu/api/entries/%v", entry["id"]) + requestResult, requestErr = executeHttpRequest(entryUrl) + if requestErr != nil { + return fmt.Errorf("failed to get entry, err: %v", requestErr) + } + + if requestResult == nil { + return fmt.Errorf("unexpected nil entry result") + } + + return nil + } + if err := retriesExecute(ShortRetriesCount, entriesCheckFunc); err != nil { + t.Errorf("%v", err) return } - entries, ok := requestResult.([]interface{}) - if !ok { - t.Errorf("invalid entries type") - return - } - - if len(entries) != entriesCount { - t.Errorf("unexpected entries result - Expected: %v, actual: %v", entriesCount, len(entries)) - return - } - - entry, ok := entries[0].(map[string]interface{}) - if !ok { - t.Errorf("invalid entry type") - return - } - - entryUrl := fmt.Sprintf("http://localhost:8899/mizu/api/entries/%v", entry["id"]) - requestResult, requestErr = ExecuteHttpRequest(entryUrl) - if requestErr != nil { - t.Errorf("failed to get entry, err: %v", requestErr) - return - } - - if requestResult == nil { - t.Errorf("unexpected nil entry result") - return - } - - fetchCmdArgs := GetDefaultFetchCommandArgs() + fetchCmdArgs := getDefaultFetchCommandArgs() fetchCmd := exec.Command(cliPath, fetchCmdArgs...) t.Logf("running command: %v", fetchCmd.String()) - t.Cleanup(func() { - if err := CleanupCommand(fetchCmd); err != nil { - t.Logf("failed to cleanup fetch command, err: %v", err) - } - }) - if err := fetchCmd.Start(); err != nil { t.Errorf("failed to start fetch command, err: %v", err) return } - time.Sleep(5 * time.Second) + harCheckFunc := func() error { + harBytes, readFileErr := ioutil.ReadFile("./unknown_source.har") + if readFileErr != nil { + return fmt.Errorf("failed to read har file, err: %v", readFileErr) + } - harBytes, readFileErr := ioutil.ReadFile("./unknown_source.har") - if readFileErr != nil { - t.Errorf("failed to read har file, err: %v", readFileErr) - return + harEntries, err := getEntriesFromHarBytes(harBytes) + if err != nil { + return fmt.Errorf("failed to get entries from har, err: %v", err) + } + + if len(harEntries) == 0 { + return fmt.Errorf("unexpected har entries result - Expected more than 0 entries") + } + + return nil } - - harEntries, err := GetEntriesFromHarBytes(harBytes) - if err != nil { - t.Errorf("failed to get entries from har, err: %v", err) - return - } - - if len(harEntries) != entriesCount { - t.Errorf("unexpected har entries result - Expected: %v, actual: %v", entriesCount, len(harEntries)) + if err := retriesExecute(ShortRetriesCount, harCheckFunc); err != nil { + t.Errorf("%v", err) return } }) diff --git a/acceptanceTests/testsUtils.go b/acceptanceTests/testsUtils.go index 123151ac3..edb2e8556 100644 --- a/acceptanceTests/testsUtils.go +++ b/acceptanceTests/testsUtils.go @@ -10,9 +10,15 @@ import ( "os/exec" "path" "syscall" + "time" ) -func GetCliPath() (string, error) { +const ( + LongRetriesCount = 100 + ShortRetriesCount = 10 +) + +func getCliPath() (string, error) { dir, filePathErr := os.Getwd() if filePathErr != nil { return "", filePathErr @@ -22,34 +28,74 @@ func GetCliPath() (string, error) { return cliPath, nil } -func GetDefaultCommandArgs() []string { +func getDefaultCommandArgs() []string { setFlag := "--set" telemetry := "telemetry=false" return []string{setFlag, telemetry} } -func GetDefaultTapCommandArgs() []string { +func getDefaultTapCommandArgs() []string { tapCommand := "tap" setFlag := "--set" namespaces := "tap.namespaces=mizu-tests" agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0" imagePullPolicy := "image-pull-policy=Never" - defaultCmdArgs := GetDefaultCommandArgs() + defaultCmdArgs := getDefaultCommandArgs() return append([]string{tapCommand, setFlag, namespaces, setFlag, agentImage, setFlag, imagePullPolicy}, defaultCmdArgs...) } -func GetDefaultFetchCommandArgs() []string { +func getDefaultFetchCommandArgs() []string { tapCommand := "fetch" - defaultCmdArgs := GetDefaultCommandArgs() + defaultCmdArgs := getDefaultCommandArgs() return append([]string{tapCommand}, defaultCmdArgs...) } -func JsonBytesToInterface(jsonBytes []byte) (interface{}, error) { +func retriesExecute(retriesCount int, executeFunc func() error) error { + var lastError error + + for i := 0; i < retriesCount; i++ { + if err := executeFunc(); err != nil { + lastError = err + + time.Sleep(1 * time.Second) + continue + } + + return nil + } + + return fmt.Errorf("reached max retries count, retries count: %v, last err: %v", retriesCount, lastError) +} + +func waitTapPodsReady() error { + resolvingUrl := fmt.Sprintf("http://localhost:8899/mizu/status/tappersCount") + tapPodsReadyFunc := func() error { + requestResult, requestErr := executeHttpRequest(resolvingUrl) + if requestErr != nil { + return requestErr + } + + tappersCount, ok := requestResult.(float64) + if !ok { + return fmt.Errorf("invalid tappers count type") + } + + if tappersCount == 0 { + return fmt.Errorf("no tappers running") + } + + return nil + } + + return retriesExecute(LongRetriesCount, tapPodsReadyFunc) +} + +func jsonBytesToInterface(jsonBytes []byte) (interface{}, error) { var result interface{} if parseErr := json.Unmarshal(jsonBytes, &result); parseErr != nil { return nil, parseErr @@ -58,7 +104,7 @@ func JsonBytesToInterface(jsonBytes []byte) (interface{}, error) { return result, nil } -func ExecuteHttpRequest(url string) (interface{}, error) { +func executeHttpRequest(url string) (interface{}, error) { response, requestErr := http.Get(url) if requestErr != nil { return nil, requestErr @@ -66,15 +112,17 @@ func ExecuteHttpRequest(url string) (interface{}, error) { return nil, fmt.Errorf("invalid status code %v", response.StatusCode) } + defer func() { response.Body.Close() }() + data, readErr := ioutil.ReadAll(response.Body) if readErr != nil { return nil, readErr } - return JsonBytesToInterface(data) + return jsonBytesToInterface(data) } -func CleanupCommand(cmd *exec.Cmd) error { +func cleanupCommand(cmd *exec.Cmd) error { if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil { return err } @@ -86,8 +134,8 @@ func CleanupCommand(cmd *exec.Cmd) error { return nil } -func GetEntriesFromHarBytes(harBytes []byte) ([]interface{}, error){ - harInterface, convertErr := JsonBytesToInterface(harBytes) +func getEntriesFromHarBytes(harBytes []byte) ([]interface{}, error){ + harInterface, convertErr := jsonBytesToInterface(harBytes) if convertErr != nil { return nil, convertErr } @@ -97,14 +145,12 @@ func GetEntriesFromHarBytes(harBytes []byte) ([]interface{}, error){ return nil, errors.New("invalid har type") } - harLogInterface := har["log"] - harLog, ok := harLogInterface.(map[string]interface{}) + harLog, ok := har["log"].(map[string]interface{}) if !ok { return nil, errors.New("invalid har log type") } - harEntriesInterface := harLog["entries"] - harEntries, ok := harEntriesInterface.([]interface{}) + harEntries, ok := harLog["entries"].([]interface{}) if !ok { return nil, errors.New("invalid har entries type") } diff --git a/agent/pkg/api/socket_server_handlers.go b/agent/pkg/api/socket_server_handlers.go index f6b4627c8..0e6a9be57 100644 --- a/agent/pkg/api/socket_server_handlers.go +++ b/agent/pkg/api/socket_server_handlers.go @@ -28,6 +28,7 @@ func init() { func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) { if isTapper { rlog.Infof("Websocket event - Tapper connected, socket ID: %d", socketId) + providers.TapperAdded() } else { rlog.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId) socketListLock.Lock() @@ -39,6 +40,7 @@ func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) { func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) { if isTapper { rlog.Infof("Websocket event - Tapper disconnected, socket ID: %d", socketId) + providers.TapperRemoved() } else { rlog.Infof("Websocket event - Browser socket disconnected, socket ID: %d", socketId) socketListLock.Lock() diff --git a/agent/pkg/controllers/status_controller.go b/agent/pkg/controllers/status_controller.go index 5e046ee82..e7f5bbc32 100644 --- a/agent/pkg/controllers/status_controller.go +++ b/agent/pkg/controllers/status_controller.go @@ -30,3 +30,7 @@ func PostTappedPods(c *gin.Context) { api.BroadcastToBrowserClients(jsonBytes) } } + +func GetTappersCount(c *gin.Context) { + c.JSON(http.StatusOK, providers.TappersCount) +} diff --git a/agent/pkg/providers/status_provider.go b/agent/pkg/providers/status_provider.go index 4a6b14468..b8971ee4b 100644 --- a/agent/pkg/providers/status_provider.go +++ b/agent/pkg/providers/status_provider.go @@ -4,14 +4,18 @@ import ( "github.com/patrickmn/go-cache" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/tap" + "sync" "time" ) const tlsLinkRetainmentTime = time.Minute * 15 var ( - TapStatus shared.TapStatus + TappersCount int + TapStatus shared.TapStatus RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime) + + tappersCountLock = sync.Mutex{} ) func GetAllRecentTLSAddresses() []string { @@ -26,3 +30,15 @@ func GetAllRecentTLSAddresses() []string { return recentTLSLinks } + +func TapperAdded() { + tappersCountLock.Lock() + TappersCount++ + tappersCountLock.Unlock() +} + +func TapperRemoved() { + tappersCountLock.Lock() + TappersCount-- + tappersCountLock.Unlock() +} diff --git a/agent/pkg/routes/status_routes.go b/agent/pkg/routes/status_routes.go index a6aa41151..4271332b4 100644 --- a/agent/pkg/routes/status_routes.go +++ b/agent/pkg/routes/status_routes.go @@ -9,4 +9,6 @@ func StatusRoutes(ginApp *gin.Engine) { routeGroup := ginApp.Group("/status") routeGroup.POST("/tappedPods", controllers.PostTappedPods) + + routeGroup.GET("/tappersCount", controllers.GetTappersCount) } From 2575ad722a4b3b619b050032e566d8c6ff9b3712 Mon Sep 17 00:00:00 2001 From: Igor Gov Date: Sun, 22 Aug 2021 11:41:38 +0300 Subject: [PATCH 03/11] Introducing API server provider (#243) --- cli/apiserver/provider.go | 168 +++++++++++++++++++++++++++++++ cli/cmd/common.go | 45 +++++++++ cli/cmd/fetch.go | 10 +- cli/cmd/fetchRunner.go | 91 ++--------------- cli/cmd/tapRunner.go | 142 ++++++++------------------ cli/cmd/viewRunner.go | 17 ++-- cli/mizu/fsUtils/zipUtils.go | 59 +++++++++++ cli/mizu/version/versionCheck.go | 28 +----- cli/telemetry/telemetry.go | 31 +----- 9 files changed, 353 insertions(+), 238 deletions(-) create mode 100644 cli/apiserver/provider.go create mode 100644 cli/cmd/common.go diff --git a/cli/apiserver/provider.go b/cli/apiserver/provider.go new file mode 100644 index 000000000..24f1b193e --- /dev/null +++ b/cli/apiserver/provider.go @@ -0,0 +1,168 @@ +package apiserver + +import ( + "archive/zip" + "bytes" + "encoding/json" + "fmt" + "github.com/up9inc/mizu/cli/logger" + "github.com/up9inc/mizu/cli/uiUtils" + "github.com/up9inc/mizu/shared" + "io/ioutil" + core "k8s.io/api/core/v1" + "net/http" + "net/url" + "time" +) + +type apiServerProvider struct { + url string + isReady bool +} + +var Provider = apiServerProvider{} + +func (provider *apiServerProvider) InitAndTestConnection(url string, retries int) error { + healthUrl := fmt.Sprintf("%s/", url) + retriesLeft := retries + for retriesLeft > 0 { + if response, err := http.Get(healthUrl); err != nil { + logger.Log.Debugf("[ERROR] failed connecting to api server %v", err) + } else if response.StatusCode != 200 { + logger.Log.Debugf("can't connect to api server yet, response status code %v", response.StatusCode) + } else { + logger.Log.Debugf("connection test to api server passed successfully") + break + } + retriesLeft -= 1 + time.Sleep(time.Second) + } + + if retriesLeft == 0 { + provider.isReady = false + return fmt.Errorf("couldn't reach the api server after %v retries", retries) + } + provider.url = url + provider.isReady = true + return nil +} + +func (provider *apiServerProvider) ReportTappedPods(pods []core.Pod) error { + if !provider.isReady { + return fmt.Errorf("trying to reach api server when not initialized yet") + } + tappedPodsUrl := fmt.Sprintf("%s/status/tappedPods", provider.url) + + podInfos := make([]shared.PodInfo, 0) + for _, pod := range pods { + podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace}) + } + tapStatus := shared.TapStatus{Pods: podInfos} + + if jsonValue, err := json.Marshal(tapStatus); err != nil { + return fmt.Errorf("failed Marshal the tapped pods %w", err) + } else { + if response, err := http.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil { + return fmt.Errorf("failed sending to API server the tapped pods %w", err) + } else if response.StatusCode != 200 { + return fmt.Errorf("failed sending to API server the tapped pods, response status code %v", response.StatusCode) + } else { + logger.Log.Debugf("Reported to server API about %d taped pods successfully", len(podInfos)) + return nil + } + } +} + +func (provider *apiServerProvider) RequestAnalysis(analysisDestination string, sleepIntervalSec int) error { + if !provider.isReady { + return fmt.Errorf("trying to reach api server when not initialized yet") + } + urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", provider.url, url.QueryEscape(analysisDestination), sleepIntervalSec) + u, parseErr := url.ParseRequestURI(urlPath) + if parseErr != nil { + logger.Log.Fatal("Failed parsing the URL (consider changing the analysis dest URL), err: %v", parseErr) + } + + logger.Log.Debugf("Analysis url %v", u.String()) + if response, requestErr := http.Get(u.String()); requestErr != nil { + return fmt.Errorf("failed to notify agent for analysis, err: %w", requestErr) + } else if response.StatusCode != 200 { + return fmt.Errorf("failed to notify agent for analysis, status code: %v", response.StatusCode) + } else { + logger.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis") + return nil + } +} + +func (provider *apiServerProvider) GetGeneralStats() (map[string]interface{}, error) { + if !provider.isReady { + return nil, fmt.Errorf("trying to reach api server when not initialized yet") + } + generalStatsUrl := fmt.Sprintf("%s/api/generalStats", provider.url) + + response, requestErr := http.Get(generalStatsUrl) + if requestErr != nil { + return nil, fmt.Errorf("failed to get general stats for telemetry, err: %w", requestErr) + } else if response.StatusCode != 200 { + return nil, fmt.Errorf("failed to get general stats for telemetry, status code: %v", response.StatusCode) + } + + defer func() { _ = response.Body.Close() }() + + data, readErr := ioutil.ReadAll(response.Body) + if readErr != nil { + return nil, fmt.Errorf("failed to read general stats for telemetry, err: %v", readErr) + } + + var generalStats map[string]interface{} + if parseErr := json.Unmarshal(data, &generalStats); parseErr != nil { + return nil, fmt.Errorf("failed to parse general stats for telemetry, err: %v", parseErr) + } + return generalStats, nil +} + +func (provider *apiServerProvider) GetHars(fromTimestamp int, toTimestamp int) (*zip.Reader, error) { + if !provider.isReady { + return nil, fmt.Errorf("trying to reach api server when not initialized yet") + } + resp, err := http.Get(fmt.Sprintf("%s/api/har?from=%v&to=%v", provider.url, fromTimestamp, toTimestamp)) + if err != nil { + return nil, fmt.Errorf("failed getting har from api server %w", err) + } + + defer func() { _ = resp.Body.Close() }() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed reading hars %w", err) + } + + zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) + if err != nil { + return nil, fmt.Errorf("failed craeting zip reader %w", err) + } + return zipReader, nil +} + +func (provider *apiServerProvider) GetVersion() (string, error) { + if !provider.isReady { + return "", fmt.Errorf("trying to reach api server when not initialized yet") + } + versionUrl, _ := url.Parse(fmt.Sprintf("%s/metadata/version", provider.url)) + req := &http.Request{ + Method: http.MethodGet, + URL: versionUrl, + } + statusResp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer statusResp.Body.Close() + + versionResponse := &shared.VersionResponse{} + if err := json.NewDecoder(statusResp.Body).Decode(&versionResponse); err != nil { + return "", err + } + + return versionResponse.SemVer, nil +} diff --git a/cli/cmd/common.go b/cli/cmd/common.go new file mode 100644 index 000000000..907663b2c --- /dev/null +++ b/cli/cmd/common.go @@ -0,0 +1,45 @@ +package cmd + +import ( + "context" + "fmt" + "github.com/up9inc/mizu/cli/config" + "github.com/up9inc/mizu/cli/config/configStructs" + "github.com/up9inc/mizu/cli/errormessage" + "github.com/up9inc/mizu/cli/kubernetes" + "github.com/up9inc/mizu/cli/logger" + "github.com/up9inc/mizu/cli/mizu" + "github.com/up9inc/mizu/cli/uiUtils" + "os" + "os/signal" + "syscall" +) + +func GetApiServerUrl() string { + return fmt.Sprintf("http://%s", kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort)) +} + +func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) { + err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName) + if err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v\n"+ + "Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName)) + cancel() + } +} + +func waitForFinish(ctx context.Context, cancel context.CancelFunc) { + logger.Log.Debugf("waiting for finish...") + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + // block until ctx cancel is called or termination signal is received + select { + case <-ctx.Done(): + logger.Log.Debugf("ctx done") + break + case <-sigChan: + logger.Log.Debugf("Got termination signal, canceling execution...") + cancel() + } +} diff --git a/cli/cmd/fetch.go b/cli/cmd/fetch.go index f18e5bf3f..dfddc350d 100644 --- a/cli/cmd/fetch.go +++ b/cli/cmd/fetch.go @@ -3,10 +3,13 @@ package cmd import ( "github.com/creasty/defaults" "github.com/spf13/cobra" + "github.com/up9inc/mizu/cli/apiserver" "github.com/up9inc/mizu/cli/config" "github.com/up9inc/mizu/cli/config/configStructs" + "github.com/up9inc/mizu/cli/logger" "github.com/up9inc/mizu/cli/mizu/version" "github.com/up9inc/mizu/cli/telemetry" + "github.com/up9inc/mizu/cli/uiUtils" ) var fetchCmd = &cobra.Command{ @@ -15,7 +18,12 @@ var fetchCmd = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { go telemetry.ReportRun("fetch", config.Config.Fetch) - if isCompatible, err := version.CheckVersionCompatibility(config.Config.Fetch.GuiPort); err != nil { + if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl(), 1); err != nil { + logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, make sure one running") + return nil + } + + if isCompatible, err := version.CheckVersionCompatibility(); err != nil { return err } else if !isCompatible { return nil diff --git a/cli/cmd/fetchRunner.go b/cli/cmd/fetchRunner.go index b0c76020e..aeff496af 100644 --- a/cli/cmd/fetchRunner.go +++ b/cli/cmd/fetchRunner.go @@ -1,96 +1,25 @@ package cmd import ( - "archive/zip" - "bytes" - "fmt" + "github.com/up9inc/mizu/cli/apiserver" "github.com/up9inc/mizu/cli/config" - "github.com/up9inc/mizu/cli/kubernetes" "github.com/up9inc/mizu/cli/logger" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "path/filepath" - "strings" + "github.com/up9inc/mizu/cli/mizu/fsUtils" + "github.com/up9inc/mizu/cli/uiUtils" ) func RunMizuFetch() { - mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Fetch.GuiPort) - resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, config.Config.Fetch.FromTimestamp, config.Config.Fetch.ToTimestamp)) - if err != nil { - log.Fatal(err) + if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl(), 5); err != nil { + logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, check logs") } - defer func() { _ = resp.Body.Close() }() - - body, err := ioutil.ReadAll(resp.Body) + zipReader, err := apiserver.Provider.GetHars(config.Config.Fetch.FromTimestamp, config.Config.Fetch.ToTimestamp) if err != nil { - log.Fatal(err) + logger.Log.Errorf("Failed fetch data from API server %v", err) + return } - zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) - if err != nil { - log.Fatal(err) + if err := fsUtils.Unzip(zipReader, config.Config.Fetch.Directory); err != nil { + logger.Log.Debugf("[ERROR] failed unzip %v", err) } - - _ = Unzip(zipReader, config.Config.Fetch.Directory) -} - -func Unzip(reader *zip.Reader, dest string) error { - dest, _ = filepath.Abs(dest) - _ = os.MkdirAll(dest, os.ModePerm) - - // Closure to address file descriptors issue with all the deferred .Close() methods - extractAndWriteFile := func(f *zip.File) error { - rc, err := f.Open() - if err != nil { - return err - } - defer func() { - if err := rc.Close(); err != nil { - panic(err) - } - }() - - path := filepath.Join(dest, f.Name) - - // Check for ZipSlip (Directory traversal) - if !strings.HasPrefix(path, filepath.Clean(dest)+string(os.PathSeparator)) { - return fmt.Errorf("illegal file path: %s", path) - } - - if f.FileInfo().IsDir() { - _ = os.MkdirAll(path, f.Mode()) - } else { - _ = os.MkdirAll(filepath.Dir(path), f.Mode()) - logger.Log.Infof("writing HAR file [ %v ]", path) - f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) - if err != nil { - return err - } - defer func() { - if err := f.Close(); err != nil { - panic(err) - } - logger.Log.Info(" done") - }() - - _, err = io.Copy(f, rc) - if err != nil { - return err - } - } - return nil - } - - for _, f := range reader.File { - err := extractAndWriteFile(f) - if err != nil { - return err - } - } - - return nil } diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 762c1222f..d19888e62 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -1,35 +1,28 @@ package cmd import ( - "bytes" "context" - "encoding/json" "fmt" + "github.com/up9inc/mizu/cli/apiserver" "github.com/up9inc/mizu/cli/config" "github.com/up9inc/mizu/cli/config/configStructs" + "github.com/up9inc/mizu/cli/errormessage" + "github.com/up9inc/mizu/cli/kubernetes" "github.com/up9inc/mizu/cli/logger" + "github.com/up9inc/mizu/cli/mizu" "github.com/up9inc/mizu/cli/mizu/fsUtils" "github.com/up9inc/mizu/cli/mizu/goUtils" "github.com/up9inc/mizu/cli/telemetry" - "net/http" - "net/url" - "os" - "os/signal" - "path" - "regexp" - "strings" - "syscall" - "time" - - "github.com/up9inc/mizu/cli/errormessage" - "github.com/up9inc/mizu/cli/kubernetes" - "github.com/up9inc/mizu/cli/mizu" "github.com/up9inc/mizu/cli/uiUtils" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/debounce" yaml "gopkg.in/yaml.v3" core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" + "path" + "regexp" + "strings" + "time" ) const ( @@ -60,7 +53,6 @@ func RunMizuTap() { return } } - kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath()) if err != nil { logger.Log.Error(err) @@ -108,13 +100,13 @@ func RunMizuTap() { nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods) - defer cleanUpMizu(kubernetesProvider) + defer finishMizuExecution(kubernetesProvider) if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions, mizuValidationRules); err != nil { logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err))) return } - go goUtils.HandleExcWrapper(createProxyToApiServerPod, ctx, kubernetesProvider, cancel) + go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel) go goUtils.HandleExcWrapper(watchPodsForTapping, ctx, kubernetesProvider, targetNamespaces, cancel) //block until exit signal or error @@ -261,23 +253,26 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi return nil } -func cleanUpMizu(kubernetesProvider *kubernetes.Provider) { - telemetry.ReportAPICalls(config.Config.Tap.GuiPort) - cleanUpMizuResources(kubernetesProvider) -} - -func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) { +func finishMizuExecution(kubernetesProvider *kubernetes.Provider) { + telemetry.ReportAPICalls() removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) defer cancel() + dumpLogsIfNeeded(kubernetesProvider, removalCtx) + cleanUpMizuResources(kubernetesProvider, removalCtx, cancel) +} - if config.Config.DumpLogs { - mizuDir := mizu.GetMizuFolderPath() - filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05"))) - if err := fsUtils.DumpLogs(kubernetesProvider, removalCtx, filePath); err != nil { - logger.Log.Errorf("Failed dump logs %v", err) - } +func dumpLogsIfNeeded(kubernetesProvider *kubernetes.Provider, removalCtx context.Context) { + if !config.Config.DumpLogs { + return } + mizuDir := mizu.GetMizuFolderPath() + filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05"))) + if err := fsUtils.DumpLogs(kubernetesProvider, removalCtx, filePath); err != nil { + logger.Log.Errorf("Failed dump logs %v", err) + } +} +func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider, removalCtx context.Context, cancel context.CancelFunc) { logger.Log.Infof("\nRemoving mizu resources\n") if !config.Config.IsNsRestrictedMode() { @@ -342,7 +337,7 @@ func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, k if err := kubernetesProvider.WaitUtilNamespaceDeleted(ctx, config.Config.MizuResourcesNamespace); err != nil { switch { case ctx.Err() == context.Canceled: - // Do nothing. User interrupted the wait. + logger.Log.Debugf("Do nothing. User interrupted the wait") case err == wait.ErrWaitTimeout: logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Timeout while removing Namespace %s", config.Config.MizuResourcesNamespace)) default: @@ -351,29 +346,6 @@ func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, k } } -func reportTappedPods() { - mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort) - tappedPodsUrl := fmt.Sprintf("http://%s/status/tappedPods", mizuProxiedUrl) - - podInfos := make([]shared.PodInfo, 0) - for _, pod := range state.currentlyTappedPods { - podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace}) - } - tapStatus := shared.TapStatus{Pods: podInfos} - - if jsonValue, err := json.Marshal(tapStatus); err != nil { - logger.Log.Debugf("[ERROR] failed Marshal the tapped pods %v", err) - } else { - if response, err := http.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil { - logger.Log.Debugf("[ERROR] failed sending to API server the tapped pods %v", err) - } else if response.StatusCode != 200 { - logger.Log.Debugf("[ERROR] failed sending to API server the tapped pods, response status code %v", response.StatusCode) - } else { - logger.Log.Debugf("Reported to server API about %d taped pods successfully", len(podInfos)) - } - } -} - func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, targetNamespaces []string, cancel context.CancelFunc) { added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, targetNamespaces, config.Config.Tap.PodRegex()) @@ -389,7 +361,9 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro return } - reportTappedPods() + if err := apiserver.Provider.ReportTappedPods(state.currentlyTappedPods); err != nil { + logger.Log.Debugf("[Error] failed update tapped pods %v", err) + } nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods) if err != nil { @@ -496,7 +470,7 @@ func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod { return missingPods } -func createProxyToApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) { +func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) { podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.ApiServerPodName)) added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex) isPodReady := false @@ -522,10 +496,17 @@ func createProxyToApiServerPod(ctx context.Context, kubernetesProvider *kubernet if modifiedPod.Status.Phase == core.PodRunning && !isPodReady { isPodReady = true go startProxyReportErrorIfAny(kubernetesProvider, cancel) - logger.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort)) - time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready - requestForAnalysis() - reportTappedPods() + + if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl(), 20); err != nil { + logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, check logs") + cancel() + break + } + logger.Log.Infof("Mizu is available at %s\n", GetApiServerUrl()) + requestForAnalysisIfNeeded() + if err := apiserver.Provider.ReportTappedPods(state.currentlyTappedPods); err != nil { + logger.Log.Debugf("[Error] failed update tapped pods %v", err) + } } case <-timeAfter: if !isPodReady { @@ -539,34 +520,12 @@ func createProxyToApiServerPod(ctx context.Context, kubernetesProvider *kubernet } } -func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) { - err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName) - if err != nil { - logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v\n"+ - "Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName)) - cancel() - } -} - -func requestForAnalysis() { +func requestForAnalysisIfNeeded() { if !config.Config.Tap.Analysis { return } - - mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort) - urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(config.Config.Tap.AnalysisDestination), config.Config.Tap.SleepIntervalSec) - u, parseErr := url.ParseRequestURI(urlPath) - if parseErr != nil { - logger.Log.Fatal("Failed parsing the URL (consider changing the analysis dest URL), err: %v", parseErr) - } - - logger.Log.Debugf("Sending get request to %v", u.String()) - if response, requestErr := http.Get(u.String()); requestErr != nil { - logger.Log.Errorf("Failed to notify agent for analysis, err: %v", requestErr) - } else if response.StatusCode != 200 { - logger.Log.Errorf("Failed to notify agent for analysis, status code: %v", response.StatusCode) - } else { - logger.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis") + if err := apiserver.Provider.RequestAnalysis(config.Config.Tap.AnalysisDestination, config.Config.Tap.SleepIntervalSec); err != nil { + logger.Log.Debugf("[Error] failed requesting for analysis %v", err) } } @@ -598,19 +557,6 @@ func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) map[string][]string { return nodeToTappedPodIPMap } -func waitForFinish(ctx context.Context, cancel context.CancelFunc) { - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - - // block until ctx cancel is called or termination signal is received - select { - case <-ctx.Done(): - break - case <-sigChan: - cancel() - } -} - func getNamespaces(kubernetesProvider *kubernetes.Provider) []string { if config.Config.Tap.AllNamespaces { return []string{mizu.K8sAllNamespaces} diff --git a/cli/cmd/viewRunner.go b/cli/cmd/viewRunner.go index 11db07187..4ef1d6dfa 100644 --- a/cli/cmd/viewRunner.go +++ b/cli/cmd/viewRunner.go @@ -3,13 +3,14 @@ package cmd import ( "context" "fmt" + "github.com/up9inc/mizu/cli/apiserver" "github.com/up9inc/mizu/cli/config" "github.com/up9inc/mizu/cli/kubernetes" "github.com/up9inc/mizu/cli/logger" "github.com/up9inc/mizu/cli/mizu" "github.com/up9inc/mizu/cli/mizu/version" + "github.com/up9inc/mizu/cli/uiUtils" "net/http" - "time" ) func runMizuView() { @@ -34,19 +35,21 @@ func runMizuView() { return } - mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.View.GuiPort) - _, err = http.Get(fmt.Sprintf("http://%s/", mizuProxiedUrl)) - if err == nil { + response, err := http.Get(fmt.Sprintf("%s/", GetApiServerUrl())) + if err == nil && response.StatusCode == 200 { logger.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, config.Config.View.GuiPort) return } logger.Log.Infof("Establishing connection to k8s cluster...") go startProxyReportErrorIfAny(kubernetesProvider, cancel) - time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready + if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl(), 10); err != nil { + logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, check logs") + return + } - logger.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.View.GuiPort)) - if isCompatible, err := version.CheckVersionCompatibility(config.Config.View.GuiPort); err != nil { + logger.Log.Infof("Mizu is available at %s\n", GetApiServerUrl()) + if isCompatible, err := version.CheckVersionCompatibility(); err != nil { logger.Log.Errorf("Failed to check versions compatibility %v", err) cancel() return diff --git a/cli/mizu/fsUtils/zipUtils.go b/cli/mizu/fsUtils/zipUtils.go index be64e644c..3c2c5fe93 100644 --- a/cli/mizu/fsUtils/zipUtils.go +++ b/cli/mizu/fsUtils/zipUtils.go @@ -3,9 +3,11 @@ package fsUtils import ( "archive/zip" "fmt" + "github.com/up9inc/mizu/cli/logger" "io" "os" "path/filepath" + "strings" ) func AddFileToZip(zipWriter *zip.Writer, filename string) error { @@ -53,3 +55,60 @@ func AddStrToZip(writer *zip.Writer, logs string, fileName string) error { } return nil } + +func Unzip(reader *zip.Reader, dest string) error { + dest, _ = filepath.Abs(dest) + _ = os.MkdirAll(dest, os.ModePerm) + + // Closure to address file descriptors issue with all the deferred .Close() methods + extractAndWriteFile := func(f *zip.File) error { + rc, err := f.Open() + if err != nil { + return err + } + defer func() { + if err := rc.Close(); err != nil { + panic(err) + } + }() + + path := filepath.Join(dest, f.Name) + + // Check for ZipSlip (Directory traversal) + if !strings.HasPrefix(path, filepath.Clean(dest)+string(os.PathSeparator)) { + return fmt.Errorf("illegal file path: %s", path) + } + + if f.FileInfo().IsDir() { + _ = os.MkdirAll(path, f.Mode()) + } else { + _ = os.MkdirAll(filepath.Dir(path), f.Mode()) + logger.Log.Infof("writing HAR file [ %v ]", path) + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return err + } + defer func() { + if err := f.Close(); err != nil { + panic(err) + } + logger.Log.Info(" done") + }() + + _, err = io.Copy(f, rc) + if err != nil { + return err + } + } + return nil + } + + for _, f := range reader.File { + err := extractAndWriteFile(f) + if err != nil { + return err + } + } + + return nil +} diff --git a/cli/mizu/version/versionCheck.go b/cli/mizu/version/versionCheck.go index eb479dd7a..f9f4d2fc5 100644 --- a/cli/mizu/version/versionCheck.go +++ b/cli/mizu/version/versionCheck.go @@ -2,43 +2,21 @@ package version import ( "context" - "encoding/json" "fmt" + "github.com/up9inc/mizu/cli/apiserver" "github.com/up9inc/mizu/cli/logger" "github.com/up9inc/mizu/cli/mizu" "io/ioutil" "net/http" - "net/url" "time" "github.com/google/go-github/v37/github" "github.com/up9inc/mizu/cli/uiUtils" - "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/semver" ) -func getApiVersion(port uint16) (string, error) { - versionUrl, _ := url.Parse(fmt.Sprintf("http://localhost:%d/mizu/metadata/version", port)) - req := &http.Request{ - Method: http.MethodGet, - URL: versionUrl, - } - statusResp, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - defer statusResp.Body.Close() - - versionResponse := &shared.VersionResponse{} - if err := json.NewDecoder(statusResp.Body).Decode(&versionResponse); err != nil { - return "", err - } - - return versionResponse.SemVer, nil -} - -func CheckVersionCompatibility(port uint16) (bool, error) { - apiSemVer, err := getApiVersion(port) +func CheckVersionCompatibility() (bool, error) { + apiSemVer, err := apiserver.Provider.GetVersion() if err != nil { return false, err } diff --git a/cli/telemetry/telemetry.go b/cli/telemetry/telemetry.go index ffa65771b..f0f9a8360 100644 --- a/cli/telemetry/telemetry.go +++ b/cli/telemetry/telemetry.go @@ -5,11 +5,10 @@ import ( "encoding/json" "fmt" "github.com/denisbrodbeck/machineid" + "github.com/up9inc/mizu/cli/apiserver" "github.com/up9inc/mizu/cli/config" - "github.com/up9inc/mizu/cli/kubernetes" "github.com/up9inc/mizu/cli/logger" "github.com/up9inc/mizu/cli/mizu" - "io/ioutil" "net/http" ) @@ -35,35 +34,15 @@ func ReportRun(cmd string, args interface{}) { logger.Log.Debugf("successfully reported telemetry for cmd %v", cmd) } -func ReportAPICalls(mizuPort uint16) { +func ReportAPICalls() { if !shouldRunTelemetry() { logger.Log.Debugf("not reporting telemetry") return } - mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizuPort) - generalStatsUrl := fmt.Sprintf("http://%s/api/generalStats", mizuProxiedUrl) - - response, requestErr := http.Get(generalStatsUrl) - if requestErr != nil { - logger.Log.Debugf("ERROR: failed to get general stats for telemetry, err: %v", requestErr) - return - } else if response.StatusCode != 200 { - logger.Log.Debugf("ERROR: failed to get general stats for telemetry, status code: %v", response.StatusCode) - return - } - - defer func() { _ = response.Body.Close() }() - - data, readErr := ioutil.ReadAll(response.Body) - if readErr != nil { - logger.Log.Debugf("ERROR: failed to read general stats for telemetry, err: %v", readErr) - return - } - - var generalStats map[string]interface{} - if parseErr := json.Unmarshal(data, &generalStats); parseErr != nil { - logger.Log.Debugf("ERROR: failed to parse general stats for telemetry, err: %v", parseErr) + generalStats, err := apiserver.Provider.GetGeneralStats() + if err != nil { + logger.Log.Debugf("[ERROR] failed get general stats from api server %v", err) return } From 35e40cd23031704fca70bd70f32f2e5bb2240b86 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Thu, 26 Aug 2021 09:56:18 +0300 Subject: [PATCH 04/11] added tap acceptance tests, fixed duplicate namespace problem (#244) --- .github/workflows/pr_validation.yml | 46 +- .github/workflows/tests_validation.yml | 56 +++ acceptanceTests/Makefile | 2 +- acceptanceTests/config_test.go | 283 +++++++++++ acceptanceTests/go.mod | 2 + acceptanceTests/go.sum | 4 + acceptanceTests/setup.sh | 13 +- acceptanceTests/tap_test.go | 662 ++++++++++++++++++++++++- acceptanceTests/testsUtils.go | 132 +++-- cli/cmd/tapRunner.go | 2 +- cli/mizu/sliceUtils.go | 14 + cli/mizu/sliceUtils_test.go | 40 ++ 12 files changed, 1151 insertions(+), 105 deletions(-) create mode 100644 .github/workflows/tests_validation.yml create mode 100644 acceptanceTests/config_test.go create mode 100644 acceptanceTests/go.sum diff --git a/.github/workflows/pr_validation.yml b/.github/workflows/pr_validation.yml index 08b8388b5..7d9e74fe7 100644 --- a/.github/workflows/pr_validation.yml +++ b/.github/workflows/pr_validation.yml @@ -1,9 +1,15 @@ name: PR validation + on: pull_request: branches: - 'develop' - 'main' + +concurrency: + group: mizu-pr-validation-${{ github.ref }} + cancel-in-progress: true + jobs: build-cli: name: Build CLI @@ -38,43 +44,3 @@ jobs: - name: Build Agent run: make agent - - run-tests-cli: - name: Run CLI tests - runs-on: ubuntu-latest - steps: - - name: Set up Go 1.16 - uses: actions/setup-go@v2 - with: - go-version: '^1.16' - - - name: Check out code into the Go module directory - uses: actions/checkout@v2 - - - name: Test - run: make test-cli - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - - run-tests-agent: - name: Run Agent tests - runs-on: ubuntu-latest - steps: - - name: Set up Go 1.16 - uses: actions/setup-go@v2 - with: - go-version: '^1.16' - - - name: Check out code into the Go module directory - uses: actions/checkout@v2 - - - shell: bash - run: | - sudo apt-get install libpcap-dev - - - name: Test - run: make test-agent - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 diff --git a/.github/workflows/tests_validation.yml b/.github/workflows/tests_validation.yml new file mode 100644 index 000000000..f63302bf7 --- /dev/null +++ b/.github/workflows/tests_validation.yml @@ -0,0 +1,56 @@ +name: tests validation + +on: + pull_request: + branches: + - 'develop' + - 'main' + push: + branches: + - 'develop' + - 'main' + +concurrency: + group: mizu-tests-validation-${{ github.ref }} + cancel-in-progress: true + +jobs: + run-tests-cli: + name: Run CLI tests + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.16 + uses: actions/setup-go@v2 + with: + go-version: '^1.16' + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Test + run: make test-cli + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + + run-tests-agent: + name: Run Agent tests + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.16 + uses: actions/setup-go@v2 + with: + go-version: '^1.16' + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - shell: bash + run: | + sudo apt-get install libpcap-dev + + - name: Test + run: make test-agent + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 diff --git a/acceptanceTests/Makefile b/acceptanceTests/Makefile index 8a142a160..c6d411544 100644 --- a/acceptanceTests/Makefile +++ b/acceptanceTests/Makefile @@ -1,2 +1,2 @@ test: ## Run acceptance tests. - @go test ./... + @go test ./... -timeout 1h diff --git a/acceptanceTests/config_test.go b/acceptanceTests/config_test.go new file mode 100644 index 000000000..248e56e9e --- /dev/null +++ b/acceptanceTests/config_test.go @@ -0,0 +1,283 @@ +package acceptanceTests + +import ( + "fmt" + "gopkg.in/yaml.v3" + "io/ioutil" + "os" + "os/exec" + "testing" +) + +type tapConfig struct { + GuiPort uint16 `yaml:"gui-port"` +} + +type configStruct struct { + Tap tapConfig `yaml:"tap"` +} + +func TestConfigRegenerate(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + configPath, configPathErr := getConfigPath() + if configPathErr != nil { + t.Errorf("failed to get config path, err: %v", cliPathErr) + return + } + + configCmdArgs := getDefaultConfigCommandArgs() + + configCmdArgs = append(configCmdArgs, "-r") + + configCmd := exec.Command(cliPath, configCmdArgs...) + t.Logf("running command: %v", configCmd.String()) + + t.Cleanup(func() { + if err := os.Remove(configPath); err != nil { + t.Logf("failed to delete config file, err: %v", err) + } + }) + + if err := configCmd.Start(); err != nil { + t.Errorf("failed to start config command, err: %v", err) + return + } + + if err := configCmd.Wait(); err != nil { + t.Errorf("failed to wait config command, err: %v", err) + return + } + + _, readFileErr := ioutil.ReadFile(configPath) + if readFileErr != nil { + t.Errorf("failed to read config file, err: %v", readFileErr) + return + } +} + +func TestConfigGuiPort(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + tests := []uint16{8898} + + for _, guiPort := range tests { + t.Run(fmt.Sprintf("%d", guiPort), func(t *testing.T) { + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + configPath, configPathErr := getConfigPath() + if configPathErr != nil { + t.Errorf("failed to get config path, err: %v", cliPathErr) + return + } + + config := configStruct{} + config.Tap.GuiPort = guiPort + + configBytes, marshalErr := yaml.Marshal(config) + if marshalErr != nil { + t.Errorf("failed to marshal config, err: %v", marshalErr) + return + } + + if writeErr := ioutil.WriteFile(configPath, configBytes, 0644); writeErr != nil { + t.Errorf("failed to write config to file, err: %v", writeErr) + return + } + + tapCmdArgs := getDefaultTapCommandArgs() + + tapNamespace := getDefaultTapNamespace() + tapCmdArgs = append(tapCmdArgs, tapNamespace...) + + tapCmd := exec.Command(cliPath, tapCmdArgs...) + t.Logf("running command: %v", tapCmd.String()) + + t.Cleanup(func() { + if err := cleanupCommand(tapCmd); err != nil { + t.Logf("failed to cleanup tap command, err: %v", err) + } + + if err := os.Remove(configPath); err != nil { + t.Logf("failed to delete config file, err: %v", err) + } + }) + + if err := tapCmd.Start(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + apiServerUrl := getApiServerUrl(guiPort) + + if err := waitTapPodsReady(apiServerUrl); err != nil { + t.Errorf("failed to start tap pods on time, err: %v", err) + return + } + }) + } +} + +func TestConfigSetGuiPort(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + tests := []struct { + ConfigFileGuiPort uint16 + SetGuiPort uint16 + }{ + {ConfigFileGuiPort: 8898, SetGuiPort: 8897}, + } + + for _, guiPortStruct := range tests { + t.Run(fmt.Sprintf("%d", guiPortStruct.SetGuiPort), func(t *testing.T) { + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + configPath, configPathErr := getConfigPath() + if configPathErr != nil { + t.Errorf("failed to get config path, err: %v", cliPathErr) + return + } + + config := configStruct{} + config.Tap.GuiPort = guiPortStruct.ConfigFileGuiPort + + configBytes, marshalErr := yaml.Marshal(config) + if marshalErr != nil { + t.Errorf("failed to marshal config, err: %v", marshalErr) + return + } + + if writeErr := ioutil.WriteFile(configPath, configBytes, 0644); writeErr != nil { + t.Errorf("failed to write config to file, err: %v", writeErr) + return + } + + tapCmdArgs := getDefaultTapCommandArgs() + + tapNamespace := getDefaultTapNamespace() + tapCmdArgs = append(tapCmdArgs, tapNamespace...) + + tapCmdArgs = append(tapCmdArgs, "--set", fmt.Sprintf("tap.gui-port=%v", guiPortStruct.SetGuiPort)) + + tapCmd := exec.Command(cliPath, tapCmdArgs...) + t.Logf("running command: %v", tapCmd.String()) + + t.Cleanup(func() { + if err := cleanupCommand(tapCmd); err != nil { + t.Logf("failed to cleanup tap command, err: %v", err) + } + + if err := os.Remove(configPath); err != nil { + t.Logf("failed to delete config file, err: %v", err) + } + }) + + if err := tapCmd.Start(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + apiServerUrl := getApiServerUrl(guiPortStruct.SetGuiPort) + + if err := waitTapPodsReady(apiServerUrl); err != nil { + t.Errorf("failed to start tap pods on time, err: %v", err) + return + } + }) + } +} + +func TestConfigFlagGuiPort(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + tests := []struct { + ConfigFileGuiPort uint16 + FlagGuiPort uint16 + }{ + {ConfigFileGuiPort: 8898, FlagGuiPort: 8896}, + } + + for _, guiPortStruct := range tests { + t.Run(fmt.Sprintf("%d", guiPortStruct.FlagGuiPort), func(t *testing.T) { + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + configPath, configPathErr := getConfigPath() + if configPathErr != nil { + t.Errorf("failed to get config path, err: %v", cliPathErr) + return + } + + config := configStruct{} + config.Tap.GuiPort = guiPortStruct.ConfigFileGuiPort + + configBytes, marshalErr := yaml.Marshal(config) + if marshalErr != nil { + t.Errorf("failed to marshal config, err: %v", marshalErr) + return + } + + if writeErr := ioutil.WriteFile(configPath, configBytes, 0644); writeErr != nil { + t.Errorf("failed to write config to file, err: %v", writeErr) + return + } + + tapCmdArgs := getDefaultTapCommandArgs() + + tapNamespace := getDefaultTapNamespace() + tapCmdArgs = append(tapCmdArgs, tapNamespace...) + + tapCmdArgs = append(tapCmdArgs, "-p", fmt.Sprintf("%v", guiPortStruct.FlagGuiPort)) + + tapCmd := exec.Command(cliPath, tapCmdArgs...) + t.Logf("running command: %v", tapCmd.String()) + + t.Cleanup(func() { + if err := cleanupCommand(tapCmd); err != nil { + t.Logf("failed to cleanup tap command, err: %v", err) + } + + if err := os.Remove(configPath); err != nil { + t.Logf("failed to delete config file, err: %v", err) + } + }) + + if err := tapCmd.Start(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + apiServerUrl := getApiServerUrl(guiPortStruct.FlagGuiPort) + + if err := waitTapPodsReady(apiServerUrl); err != nil { + t.Errorf("failed to start tap pods on time, err: %v", err) + return + } + }) + } +} diff --git a/acceptanceTests/go.mod b/acceptanceTests/go.mod index bc529ac9d..071a91936 100644 --- a/acceptanceTests/go.mod +++ b/acceptanceTests/go.mod @@ -1,3 +1,5 @@ module github.com/up9inc/mizu/tests go 1.16 + +require gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b diff --git a/acceptanceTests/go.sum b/acceptanceTests/go.sum new file mode 100644 index 000000000..e387ff0b1 --- /dev/null +++ b/acceptanceTests/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/acceptanceTests/setup.sh b/acceptanceTests/setup.sh index 0ed9dbce1..d95b72eea 100644 --- a/acceptanceTests/setup.sh +++ b/acceptanceTests/setup.sh @@ -26,14 +26,21 @@ fi echo "Starting minikube..." minikube start -echo "Creating mizu tests namespace" +echo "Creating mizu tests namespaces" kubectl create namespace mizu-tests +kubectl create namespace mizu-tests2 -echo "Creating httpbin deployment" +echo "Creating httpbin deployments" kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests +kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests -echo "Creating httpbin service" +kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2 + +echo "Creating httpbin services" kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests +kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests + +kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2 echo "Starting proxy" kubectl proxy --port=8080 & diff --git a/acceptanceTests/tap_test.go b/acceptanceTests/tap_test.go index 1e6ca9a4f..8d921ed1f 100644 --- a/acceptanceTests/tap_test.go +++ b/acceptanceTests/tap_test.go @@ -1,9 +1,13 @@ package acceptanceTests import ( + "bytes" + "encoding/json" "fmt" "io/ioutil" + "net/http" "os/exec" + "strings" "testing" "time" ) @@ -24,6 +28,10 @@ func TestTapAndFetch(t *testing.T) { } tapCmdArgs := getDefaultTapCommandArgs() + + tapNamespace := getDefaultTapNamespace() + tapCmdArgs = append(tapCmdArgs, tapNamespace...) + tapCmd := exec.Command(cliPath, tapCmdArgs...) t.Logf("running command: %v", tapCmd.String()) @@ -38,14 +46,16 @@ func TestTapAndFetch(t *testing.T) { return } - if err := waitTapPodsReady(); err != nil { + apiServerUrl := getApiServerUrl(defaultApiServerPort) + + if err := waitTapPodsReady(apiServerUrl); err != nil { t.Errorf("failed to start tap pods on time, err: %v", err) return } - proxyUrl := "http://localhost:8080/api/v1/namespaces/mizu-tests/services/httpbin/proxy/get" + proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName) for i := 0; i < entriesCount; i++ { - if _, requestErr := executeHttpRequest(proxyUrl); requestErr != nil { + if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil { t.Errorf("failed to send proxy request, err: %v", requestErr) return } @@ -54,28 +64,22 @@ func TestTapAndFetch(t *testing.T) { entriesCheckFunc := func() error { timestamp := time.Now().UnixNano() / int64(time.Millisecond) - entriesUrl := fmt.Sprintf("http://localhost:8899/mizu/api/entries?limit=%v&operator=lt×tamp=%v", entriesCount, timestamp) - requestResult, requestErr := executeHttpRequest(entriesUrl) + entriesUrl := fmt.Sprintf("%v/api/entries?limit=%v&operator=lt×tamp=%v", apiServerUrl, entriesCount, timestamp) + requestResult, requestErr := executeHttpGetRequest(entriesUrl) if requestErr != nil { return fmt.Errorf("failed to get entries, err: %v", requestErr) } - entries, ok := requestResult.([]interface{}) - if !ok { - return fmt.Errorf("invalid entries type") - } + entries := requestResult.([]interface{}) if len(entries) == 0 { return fmt.Errorf("unexpected entries result - Expected more than 0 entries") } - entry, ok := entries[0].(map[string]interface{}) - if !ok { - return fmt.Errorf("invalid entry type") - } + entry := entries[0].(map[string]interface{}) - entryUrl := fmt.Sprintf("http://localhost:8899/mizu/api/entries/%v", entry["id"]) - requestResult, requestErr = executeHttpRequest(entryUrl) + entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, entry["id"]) + requestResult, requestErr = executeHttpGetRequest(entryUrl) if requestErr != nil { return fmt.Errorf("failed to get entry, err: %v", requestErr) } @@ -86,7 +90,7 @@ func TestTapAndFetch(t *testing.T) { return nil } - if err := retriesExecute(ShortRetriesCount, entriesCheckFunc); err != nil { + if err := retriesExecute(shortRetriesCount, entriesCheckFunc); err != nil { t.Errorf("%v", err) return } @@ -117,10 +121,634 @@ func TestTapAndFetch(t *testing.T) { return nil } - if err := retriesExecute(ShortRetriesCount, harCheckFunc); err != nil { + if err := retriesExecute(shortRetriesCount, harCheckFunc); err != nil { t.Errorf("%v", err) return } }) } } + +func TestTapGuiPort(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + tests := []uint16{8898} + + for _, guiPort := range tests { + t.Run(fmt.Sprintf("%d", guiPort), func(t *testing.T) { + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + tapCmdArgs := getDefaultTapCommandArgs() + + tapNamespace := getDefaultTapNamespace() + tapCmdArgs = append(tapCmdArgs, tapNamespace...) + + tapCmdArgs = append(tapCmdArgs, "-p", fmt.Sprintf("%d", guiPort)) + + tapCmd := exec.Command(cliPath, tapCmdArgs...) + t.Logf("running command: %v", tapCmd.String()) + + t.Cleanup(func() { + if err := cleanupCommand(tapCmd); err != nil { + t.Logf("failed to cleanup tap command, err: %v", err) + } + }) + + if err := tapCmd.Start(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + apiServerUrl := getApiServerUrl(guiPort) + + if err := waitTapPodsReady(apiServerUrl); err != nil { + t.Errorf("failed to start tap pods on time, err: %v", err) + return + } + }) + } +} + +func TestTapAllNamespaces(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + expectedPods := []struct{ + Name string + Namespace string + }{ + {Name: "httpbin", Namespace: "mizu-tests"}, + {Name: "httpbin", Namespace: "mizu-tests2"}, + } + + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + tapCmdArgs := getDefaultTapCommandArgs() + tapCmdArgs = append(tapCmdArgs, "-A") + + tapCmd := exec.Command(cliPath, tapCmdArgs...) + t.Logf("running command: %v", tapCmd.String()) + + t.Cleanup(func() { + if err := cleanupCommand(tapCmd); err != nil { + t.Logf("failed to cleanup tap command, err: %v", err) + } + }) + + if err := tapCmd.Start(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + apiServerUrl := getApiServerUrl(defaultApiServerPort) + + if err := waitTapPodsReady(apiServerUrl); err != nil { + t.Errorf("failed to start tap pods on time, err: %v", err) + return + } + + podsUrl := fmt.Sprintf("%v/api/tapStatus", apiServerUrl) + requestResult, requestErr := executeHttpGetRequest(podsUrl) + if requestErr != nil { + t.Errorf("failed to get tap status, err: %v", requestErr) + return + } + + pods, err := getPods(requestResult) + if err != nil { + t.Errorf("failed to get pods, err: %v", err) + return + } + + for _, expectedPod := range expectedPods { + podFound := false + + for _, pod := range pods { + podNamespace := pod["namespace"].(string) + podName := pod["name"].(string) + + if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) { + podFound = true + break + } + } + + if !podFound { + t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name) + return + } + } +} + +func TestTapMultipleNamespaces(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + expectedPods := []struct{ + Name string + Namespace string + }{ + {Name: "httpbin", Namespace: "mizu-tests"}, + {Name: "httpbin2", Namespace: "mizu-tests"}, + {Name: "httpbin", Namespace: "mizu-tests2"}, + } + + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + tapCmdArgs := getDefaultTapCommandArgs() + var namespacesCmd []string + for _, expectedPod := range expectedPods { + namespacesCmd = append(namespacesCmd, "-n", expectedPod.Namespace) + } + tapCmdArgs = append(tapCmdArgs, namespacesCmd...) + + tapCmd := exec.Command(cliPath, tapCmdArgs...) + t.Logf("running command: %v", tapCmd.String()) + + t.Cleanup(func() { + if err := cleanupCommand(tapCmd); err != nil { + t.Logf("failed to cleanup tap command, err: %v", err) + } + }) + + if err := tapCmd.Start(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + apiServerUrl := getApiServerUrl(defaultApiServerPort) + + if err := waitTapPodsReady(apiServerUrl); err != nil { + t.Errorf("failed to start tap pods on time, err: %v", err) + return + } + + podsUrl := fmt.Sprintf("%v/api/tapStatus", apiServerUrl) + requestResult, requestErr := executeHttpGetRequest(podsUrl) + if requestErr != nil { + t.Errorf("failed to get tap status, err: %v", requestErr) + return + } + + pods, err := getPods(requestResult) + if err != nil { + t.Errorf("failed to get pods, err: %v", err) + return + } + + if len(expectedPods) != len(pods) { + t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods)) + return + } + + for _, expectedPod := range expectedPods { + podFound := false + + for _, pod := range pods { + podNamespace := pod["namespace"].(string) + podName := pod["name"].(string) + + if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) { + podFound = true + break + } + } + + if !podFound { + t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name) + return + } + } +} + +func TestTapRegex(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + regexPodName := "httpbin2" + expectedPods := []struct{ + Name string + Namespace string + }{ + {Name: regexPodName, Namespace: "mizu-tests"}, + } + + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + tapCmdArgs := getDefaultTapCommandArgsWithRegex(regexPodName) + + tapNamespace := getDefaultTapNamespace() + tapCmdArgs = append(tapCmdArgs, tapNamespace...) + + tapCmd := exec.Command(cliPath, tapCmdArgs...) + t.Logf("running command: %v", tapCmd.String()) + + t.Cleanup(func() { + if err := cleanupCommand(tapCmd); err != nil { + t.Logf("failed to cleanup tap command, err: %v", err) + } + }) + + if err := tapCmd.Start(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + apiServerUrl := getApiServerUrl(defaultApiServerPort) + + if err := waitTapPodsReady(apiServerUrl); err != nil { + t.Errorf("failed to start tap pods on time, err: %v", err) + return + } + + podsUrl := fmt.Sprintf("%v/api/tapStatus", apiServerUrl) + requestResult, requestErr := executeHttpGetRequest(podsUrl) + if requestErr != nil { + t.Errorf("failed to get tap status, err: %v", requestErr) + return + } + + pods, err := getPods(requestResult) + if err != nil { + t.Errorf("failed to get pods, err: %v", err) + return + } + + if len(expectedPods) != len(pods) { + t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods)) + return + } + + for _, expectedPod := range expectedPods { + podFound := false + + for _, pod := range pods { + podNamespace := pod["namespace"].(string) + podName := pod["name"].(string) + + if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) { + podFound = true + break + } + } + + if !podFound { + t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name) + return + } + } +} + +func TestTapDryRun(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + tapCmdArgs := getDefaultTapCommandArgs() + + tapNamespace := getDefaultTapNamespace() + tapCmdArgs = append(tapCmdArgs, tapNamespace...) + + tapCmdArgs = append(tapCmdArgs, "--dry-run") + + tapCmd := exec.Command(cliPath, tapCmdArgs...) + t.Logf("running command: %v", tapCmd.String()) + + if err := tapCmd.Start(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + resultChannel := make(chan string, 1) + + go func() { + if err := tapCmd.Wait(); err != nil { + resultChannel <- "fail" + return + } + resultChannel <- "success" + }() + + go func() { + time.Sleep(shortRetriesCount * time.Second) + resultChannel <- "fail" + }() + + testResult := <- resultChannel + if testResult != "success" { + t.Errorf("unexpected result - dry run cmd not done") + } +} + +func TestTapRedact(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + tapCmdArgs := getDefaultTapCommandArgs() + + tapNamespace := getDefaultTapNamespace() + tapCmdArgs = append(tapCmdArgs, tapNamespace...) + + tapCmd := exec.Command(cliPath, tapCmdArgs...) + t.Logf("running command: %v", tapCmd.String()) + + t.Cleanup(func() { + if err := cleanupCommand(tapCmd); err != nil { + t.Logf("failed to cleanup tap command, err: %v", err) + } + }) + + if err := tapCmd.Start(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + apiServerUrl := getApiServerUrl(defaultApiServerPort) + + if err := waitTapPodsReady(apiServerUrl); err != nil { + t.Errorf("failed to start tap pods on time, err: %v", err) + return + } + + proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName) + requestBody := map[string]string{"User": "Mizu"} + for i := 0; i < defaultEntriesCount; i++ { + if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil { + t.Errorf("failed to send proxy request, err: %v", requestErr) + return + } + } + + redactCheckFunc := func() error { + timestamp := time.Now().UnixNano() / int64(time.Millisecond) + + entriesUrl := fmt.Sprintf("%v/api/entries?limit=%v&operator=lt×tamp=%v", apiServerUrl, defaultEntriesCount, timestamp) + requestResult, requestErr := executeHttpGetRequest(entriesUrl) + if requestErr != nil { + return fmt.Errorf("failed to get entries, err: %v", requestErr) + } + + entries := requestResult.([]interface{}) + firstEntry := entries[0].(map[string]interface{}) + + entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, firstEntry["id"]) + requestResult, requestErr = executeHttpGetRequest(entryUrl) + if requestErr != nil { + return fmt.Errorf("failed to get entry, err: %v", requestErr) + } + + entry := requestResult.(map[string]interface{})["entry"].(map[string]interface{}) + entryRequest := entry["request"].(map[string]interface{}) + + headers := entryRequest["headers"].([]interface{}) + for _, headerInterface := range headers { + header := headerInterface.(map[string]interface{}) + if header["name"].(string) != "User-Agent" { + continue + } + + userAgent := header["value"].(string) + if userAgent != "[REDACTED]" { + return fmt.Errorf("unexpected result - user agent is not redacted") + } + } + + data := entryRequest["postData"].(map[string]interface{}) + textDataStr := data["text"].(string) + + var textData map[string]string + if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil { + return fmt.Errorf("failed to parse text data, err: %v", parseErr) + } + + if textData["User"] != "[REDACTED]" { + return fmt.Errorf("unexpected result - user in body is not redacted") + } + + return nil + } + if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil { + t.Errorf("%v", err) + return + } +} + +func TestTapNoRedact(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + tapCmdArgs := getDefaultTapCommandArgs() + + tapNamespace := getDefaultTapNamespace() + tapCmdArgs = append(tapCmdArgs, tapNamespace...) + + tapCmdArgs = append(tapCmdArgs, "--no-redact") + + tapCmd := exec.Command(cliPath, tapCmdArgs...) + t.Logf("running command: %v", tapCmd.String()) + + t.Cleanup(func() { + if err := cleanupCommand(tapCmd); err != nil { + t.Logf("failed to cleanup tap command, err: %v", err) + } + }) + + if err := tapCmd.Start(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + apiServerUrl := getApiServerUrl(defaultApiServerPort) + + if err := waitTapPodsReady(apiServerUrl); err != nil { + t.Errorf("failed to start tap pods on time, err: %v", err) + return + } + + proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName) + requestBody := map[string]string{"User": "Mizu"} + for i := 0; i < defaultEntriesCount; i++ { + if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil { + t.Errorf("failed to send proxy request, err: %v", requestErr) + return + } + } + + redactCheckFunc := func() error { + timestamp := time.Now().UnixNano() / int64(time.Millisecond) + + entriesUrl := fmt.Sprintf("%v/api/entries?limit=%v&operator=lt×tamp=%v", apiServerUrl, defaultEntriesCount, timestamp) + requestResult, requestErr := executeHttpGetRequest(entriesUrl) + if requestErr != nil { + return fmt.Errorf("failed to get entries, err: %v", requestErr) + } + + entries := requestResult.([]interface{}) + firstEntry := entries[0].(map[string]interface{}) + + entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, firstEntry["id"]) + requestResult, requestErr = executeHttpGetRequest(entryUrl) + if requestErr != nil { + return fmt.Errorf("failed to get entry, err: %v", requestErr) + } + + entry := requestResult.(map[string]interface{})["entry"].(map[string]interface{}) + entryRequest := entry["request"].(map[string]interface{}) + + headers := entryRequest["headers"].([]interface{}) + for _, headerInterface := range headers { + header := headerInterface.(map[string]interface{}) + if header["name"].(string) != "User-Agent" { + continue + } + + userAgent := header["value"].(string) + if userAgent == "[REDACTED]" { + return fmt.Errorf("unexpected result - user agent is redacted") + } + } + + data := entryRequest["postData"].(map[string]interface{}) + textDataStr := data["text"].(string) + + var textData map[string]string + if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil { + return fmt.Errorf("failed to parse text data, err: %v", parseErr) + } + + if textData["User"] == "[REDACTED]" { + return fmt.Errorf("unexpected result - user in body is redacted") + } + + return nil + } + if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil { + t.Errorf("%v", err) + return + } +} + +func TestTapRegexMasking(t *testing.T) { + if testing.Short() { + t.Skip("ignored acceptance test") + } + + cliPath, cliPathErr := getCliPath() + if cliPathErr != nil { + t.Errorf("failed to get cli path, err: %v", cliPathErr) + return + } + + tapCmdArgs := getDefaultTapCommandArgs() + + tapNamespace := getDefaultTapNamespace() + tapCmdArgs = append(tapCmdArgs, tapNamespace...) + + tapCmdArgs = append(tapCmdArgs, "-r", "Mizu") + + tapCmd := exec.Command(cliPath, tapCmdArgs...) + t.Logf("running command: %v", tapCmd.String()) + + t.Cleanup(func() { + if err := cleanupCommand(tapCmd); err != nil { + t.Logf("failed to cleanup tap command, err: %v", err) + } + }) + + if err := tapCmd.Start(); err != nil { + t.Errorf("failed to start tap command, err: %v", err) + return + } + + apiServerUrl := getApiServerUrl(defaultApiServerPort) + + if err := waitTapPodsReady(apiServerUrl); err != nil { + t.Errorf("failed to start tap pods on time, err: %v", err) + return + } + + proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName) + for i := 0; i < defaultEntriesCount; i++ { + response, requestErr := http.Post(fmt.Sprintf("%v/post", proxyUrl), "text/plain", bytes.NewBufferString("Mizu")) + if _, requestErr = executeHttpRequest(response, requestErr); requestErr != nil { + t.Errorf("failed to send proxy request, err: %v", requestErr) + return + } + } + + redactCheckFunc := func() error { + timestamp := time.Now().UnixNano() / int64(time.Millisecond) + + entriesUrl := fmt.Sprintf("%v/api/entries?limit=%v&operator=lt×tamp=%v", apiServerUrl, defaultEntriesCount, timestamp) + requestResult, requestErr := executeHttpGetRequest(entriesUrl) + if requestErr != nil { + return fmt.Errorf("failed to get entries, err: %v", requestErr) + } + + entries := requestResult.([]interface{}) + firstEntry := entries[0].(map[string]interface{}) + + entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, firstEntry["id"]) + requestResult, requestErr = executeHttpGetRequest(entryUrl) + if requestErr != nil { + return fmt.Errorf("failed to get entry, err: %v", requestErr) + } + + entry := requestResult.(map[string]interface{})["entry"].(map[string]interface{}) + entryRequest := entry["request"].(map[string]interface{}) + + data := entryRequest["postData"].(map[string]interface{}) + textData := data["text"].(string) + + if textData != "[REDACTED]" { + return fmt.Errorf("unexpected result - body is not redacted") + } + + return nil + } + if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil { + t.Errorf("%v", err) + return + } +} diff --git a/acceptanceTests/testsUtils.go b/acceptanceTests/testsUtils.go index edb2e8556..0d85a7353 100644 --- a/acceptanceTests/testsUtils.go +++ b/acceptanceTests/testsUtils.go @@ -1,8 +1,8 @@ package acceptanceTests import ( + "bytes" "encoding/json" - "errors" "fmt" "io/ioutil" "net/http" @@ -14,8 +14,12 @@ import ( ) const ( - LongRetriesCount = 100 - ShortRetriesCount = 10 + longRetriesCount = 100 + shortRetriesCount = 10 + defaultApiServerPort = 8899 + defaultNamespaceName = "mizu-tests" + defaultServiceName = "httpbin" + defaultEntriesCount = 50 ) func getCliPath() (string, error) { @@ -28,33 +32,64 @@ func getCliPath() (string, error) { return cliPath, nil } +func getConfigPath() (string, error) { + home, homeDirErr := os.UserHomeDir() + if homeDirErr != nil { + return "", homeDirErr + } + + return path.Join(home, ".mizu", "config.yaml"), nil +} + +func getProxyUrl(namespace string, service string) string { + return fmt.Sprintf("http://localhost:8080/api/v1/namespaces/%v/services/%v/proxy", namespace, service) +} + +func getApiServerUrl(port uint16) string { + return fmt.Sprintf("http://localhost:%v/mizu", port) +} + func getDefaultCommandArgs() []string { setFlag := "--set" telemetry := "telemetry=false" + agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0" + imagePullPolicy := "image-pull-policy=Never" - return []string{setFlag, telemetry} + return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy} } func getDefaultTapCommandArgs() []string { tapCommand := "tap" - setFlag := "--set" - namespaces := "tap.namespaces=mizu-tests" - agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0" - imagePullPolicy := "image-pull-policy=Never" - - defaultCmdArgs := getDefaultCommandArgs() - - return append([]string{tapCommand, setFlag, namespaces, setFlag, agentImage, setFlag, imagePullPolicy}, defaultCmdArgs...) -} - -func getDefaultFetchCommandArgs() []string { - tapCommand := "fetch" - defaultCmdArgs := getDefaultCommandArgs() return append([]string{tapCommand}, defaultCmdArgs...) } +func getDefaultTapCommandArgsWithRegex(regex string) []string { + tapCommand := "tap" + defaultCmdArgs := getDefaultCommandArgs() + + return append([]string{tapCommand, regex}, defaultCmdArgs...) +} + +func getDefaultTapNamespace() []string { + return []string{"-n", "mizu-tests"} +} + +func getDefaultFetchCommandArgs() []string { + fetchCommand := "fetch" + defaultCmdArgs := getDefaultCommandArgs() + + return append([]string{fetchCommand}, defaultCmdArgs...) +} + +func getDefaultConfigCommandArgs() []string { + configCommand := "config" + defaultCmdArgs := getDefaultCommandArgs() + + return append([]string{configCommand}, defaultCmdArgs...) +} + func retriesExecute(retriesCount int, executeFunc func() error) error { var lastError error @@ -72,19 +107,15 @@ func retriesExecute(retriesCount int, executeFunc func() error) error { return fmt.Errorf("reached max retries count, retries count: %v, last err: %v", retriesCount, lastError) } -func waitTapPodsReady() error { - resolvingUrl := fmt.Sprintf("http://localhost:8899/mizu/status/tappersCount") +func waitTapPodsReady(apiServerUrl string) error { + resolvingUrl := fmt.Sprintf("%v/status/tappersCount", apiServerUrl) tapPodsReadyFunc := func() error { - requestResult, requestErr := executeHttpRequest(resolvingUrl) + requestResult, requestErr := executeHttpGetRequest(resolvingUrl) if requestErr != nil { return requestErr } - tappersCount, ok := requestResult.(float64) - if !ok { - return fmt.Errorf("invalid tappers count type") - } - + tappersCount := requestResult.(float64) if tappersCount == 0 { return fmt.Errorf("no tappers running") } @@ -92,7 +123,7 @@ func waitTapPodsReady() error { return nil } - return retriesExecute(LongRetriesCount, tapPodsReadyFunc) + return retriesExecute(longRetriesCount, tapPodsReadyFunc) } func jsonBytesToInterface(jsonBytes []byte) (interface{}, error) { @@ -104,8 +135,7 @@ func jsonBytesToInterface(jsonBytes []byte) (interface{}, error) { return result, nil } -func executeHttpRequest(url string) (interface{}, error) { - response, requestErr := http.Get(url) +func executeHttpRequest(response *http.Response, requestErr error) (interface{}, error) { if requestErr != nil { return nil, requestErr } else if response.StatusCode != 200 { @@ -122,6 +152,21 @@ func executeHttpRequest(url string) (interface{}, error) { return jsonBytesToInterface(data) } +func executeHttpGetRequest(url string) (interface{}, error) { + response, requestErr := http.Get(url) + return executeHttpRequest(response, requestErr) +} + +func executeHttpPostRequest(url string, body interface{}) (interface{}, error) { + requestBody, jsonErr := json.Marshal(body) + if jsonErr != nil { + return nil, jsonErr + } + + response, requestErr := http.Post(url, "application/json", bytes.NewBuffer(requestBody)) + return executeHttpRequest(response, requestErr) +} + func cleanupCommand(cmd *exec.Cmd) error { if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil { return err @@ -134,26 +179,27 @@ func cleanupCommand(cmd *exec.Cmd) error { return nil } -func getEntriesFromHarBytes(harBytes []byte) ([]interface{}, error){ +func getEntriesFromHarBytes(harBytes []byte) ([]interface{}, error) { harInterface, convertErr := jsonBytesToInterface(harBytes) if convertErr != nil { return nil, convertErr } - har, ok := harInterface.(map[string]interface{}) - if !ok { - return nil, errors.New("invalid har type") - } - - harLog, ok := har["log"].(map[string]interface{}) - if !ok { - return nil, errors.New("invalid har log type") - } - - harEntries, ok := harLog["entries"].([]interface{}) - if !ok { - return nil, errors.New("invalid har entries type") - } + har := harInterface.(map[string]interface{}) + harLog := har["log"].(map[string]interface{}) + harEntries := harLog["entries"].([]interface{}) return harEntries, nil } + +func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) { + tapStatus := tapStatusInterface.(map[string]interface{}) + podsInterface := tapStatus["pods"].([]interface{}) + + var pods []map[string]interface{} + for _, podInterface := range podsInterface { + pods = append(pods, podInterface.(map[string]interface{})) + } + + return pods, nil +} diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index d19888e62..5c44b9348 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -561,7 +561,7 @@ func getNamespaces(kubernetesProvider *kubernetes.Provider) []string { if config.Config.Tap.AllNamespaces { return []string{mizu.K8sAllNamespaces} } else if len(config.Config.Tap.Namespaces) > 0 { - return config.Config.Tap.Namespaces + return mizu.Unique(config.Config.Tap.Namespaces) } else { return []string{kubernetesProvider.CurrentNamespace()} } diff --git a/cli/mizu/sliceUtils.go b/cli/mizu/sliceUtils.go index 551e12603..94e253225 100644 --- a/cli/mizu/sliceUtils.go +++ b/cli/mizu/sliceUtils.go @@ -9,3 +9,17 @@ func Contains(slice []string, containsValue string) bool { return false } + +func Unique(slice []string) []string { + keys := make(map[string]bool) + var list []string + + for _, entry := range slice { + if _, value := keys[entry]; !value { + keys[entry] = true + list = append(list, entry) + } + } + + return list +} diff --git a/cli/mizu/sliceUtils_test.go b/cli/mizu/sliceUtils_test.go index 49787c64c..d5e7efe6d 100644 --- a/cli/mizu/sliceUtils_test.go +++ b/cli/mizu/sliceUtils_test.go @@ -1,7 +1,9 @@ package mizu_test import ( + "fmt" "github.com/up9inc/mizu/cli/mizu" + "reflect" "testing" ) @@ -88,3 +90,41 @@ func TestContainsNilSlice(t *testing.T) { }) } } + +func TestUniqueNoDuplicateValues(t *testing.T) { + tests := []struct { + Slice []string + Expected []string + }{ + {Slice: []string{"apple", "orange", "banana", "grapes"}, Expected: []string{"apple", "orange", "banana", "grapes"}}, + {Slice: []string{"dog", "cat", "mouse"}, Expected: []string{"dog", "cat", "mouse"}}, + } + + for index, test := range tests { + t.Run(fmt.Sprintf("%v", index), func(t *testing.T) { + actual := mizu.Unique(test.Slice) + if !reflect.DeepEqual(test.Expected, actual) { + t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual) + } + }) + } +} + +func TestUniqueDuplicateValues(t *testing.T) { + tests := []struct { + Slice []string + Expected []string + }{ + {Slice: []string{"apple", "apple", "orange", "orange", "banana", "banana", "grapes", "grapes"}, Expected: []string{"apple", "orange", "banana", "grapes"}}, + {Slice: []string{"dog", "cat", "cat", "mouse"}, Expected: []string{"dog", "cat", "mouse"}}, + } + + for index, test := range tests { + t.Run(fmt.Sprintf("%v", index), func(t *testing.T) { + actual := mizu.Unique(test.Slice) + if !reflect.DeepEqual(test.Expected, actual) { + t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual) + } + }) + } +} From a9e92b60f5a4baafd89ee23d622117831732ccb3 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Thu, 26 Aug 2021 13:50:41 +0300 Subject: [PATCH 05/11] added custom config path option (#247) --- cli/cmd/config.go | 10 +++++----- cli/cmd/root.go | 5 +++++ cli/config/config.go | 26 ++++++++++++++------------ cli/config/configStruct.go | 4 ++++ cli/config/config_test.go | 4 +++- cli/mizu/fsUtils/mizuLogsUtils.go | 4 ++-- 6 files changed, 33 insertions(+), 20 deletions(-) diff --git a/cli/cmd/config.go b/cli/cmd/config.go index 2b30a9630..0a7815d9a 100644 --- a/cli/cmd/config.go +++ b/cli/cmd/config.go @@ -25,11 +25,11 @@ var configCmd = &cobra.Command{ } if config.Config.Config.Regenerate { data := []byte(template) - if err := ioutil.WriteFile(config.GetConfigFilePath(), data, 0644); err != nil { + if err := ioutil.WriteFile(config.Config.ConfigFilePath, data, 0644); err != nil { logger.Log.Errorf("Failed writing config %v", err) return nil } - logger.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, config.GetConfigFilePath()))) + logger.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, config.Config.ConfigFilePath))) } else { logger.Log.Debugf("Writing template config.\n%v", template) fmt.Printf("%v", template) @@ -41,8 +41,8 @@ var configCmd = &cobra.Command{ func init() { rootCmd.AddCommand(configCmd) - defaultConfigConfig := configStructs.ConfigConfig{} - defaults.Set(&defaultConfigConfig) + defaultConfig := config.ConfigStruct{} + defaults.Set(&defaultConfig) - configCmd.Flags().BoolP(configStructs.RegenerateConfigName, "r", defaultConfigConfig.Regenerate, fmt.Sprintf("Regenerate the config file with default values %s", config.GetConfigFilePath())) + configCmd.Flags().BoolP(configStructs.RegenerateConfigName, "r", defaultConfig.Config.Regenerate, fmt.Sprintf("Regenerate the config file with default values to path %s or to chosen path using --%s", defaultConfig.ConfigFilePath, config.ConfigFilePathCommandName)) } diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 3a1f4d3e0..d14c54acf 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "github.com/creasty/defaults" "github.com/spf13/cobra" "github.com/up9inc/mizu/cli/config" "github.com/up9inc/mizu/cli/logger" @@ -25,7 +26,11 @@ Further info is available at https://github.com/up9inc/mizu`, } func init() { + defaultConfig := config.ConfigStruct{} + defaults.Set(&defaultConfig) + rootCmd.PersistentFlags().StringSlice(config.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", config.SetCommandName)) + rootCmd.PersistentFlags().String(config.ConfigFilePathCommandName, defaultConfig.ConfigFilePath, fmt.Sprintf("Override config file path using --%s", config.ConfigFilePathCommandName)) } func printNewVersionIfNeeded(versionChan chan string) { diff --git a/cli/config/config.go b/cli/config/config.go index e1f66c036..ab6332b51 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -7,7 +7,6 @@ import ( "github.com/up9inc/mizu/cli/mizu" "io/ioutil" "os" - "path" "reflect" "strconv" "strings" @@ -27,7 +26,7 @@ const ( ) var ( - Config = ConfigStruct{} + Config = ConfigStruct{} cmdName string ) @@ -38,9 +37,11 @@ func InitConfig(cmd *cobra.Command) error { return err } - if err := mergeConfigFile(); err != nil { - return fmt.Errorf("invalid config, %w\n" + - "you can regenerate the file by removing it (%v) and using `mizu config -r`", err, GetConfigFilePath()) + configFilePath := cmd.Flags().Lookup(ConfigFilePathCommandName).Value.String() + + if err := mergeConfigFile(configFilePath); err != nil { + return fmt.Errorf("invalid config, %w\n"+ + "you can regenerate the file by removing it (%v) and using `mizu config -r`", err, configFilePath) } cmd.Flags().Visit(initFlag) @@ -63,12 +64,8 @@ func GetConfigWithDefaults() (string, error) { return uiUtils.PrettyYaml(defaultConf) } -func GetConfigFilePath() string { - return path.Join(mizu.GetMizuFolderPath(), "config.yaml") -} - -func mergeConfigFile() error { - reader, openErr := os.Open(GetConfigFilePath()) +func mergeConfigFile(configFilePath string) error { + reader, openErr := os.Open(configFilePath) if openErr != nil { return nil } @@ -89,7 +86,12 @@ func mergeConfigFile() error { func initFlag(f *pflag.Flag) { configElemValue := reflect.ValueOf(&Config).Elem() - flagPath := []string {cmdName, f.Name} + var flagPath []string + if mizu.Contains([]string{ConfigFilePathCommandName}, f.Name) { + flagPath = []string{f.Name} + } else { + flagPath = []string{cmdName, f.Name} + } sliceValue, isSliceValue := f.Value.(pflag.SliceValue) if !isSliceValue { diff --git a/cli/config/configStruct.go b/cli/config/configStruct.go index aa4de7944..bf3472adc 100644 --- a/cli/config/configStruct.go +++ b/cli/config/configStruct.go @@ -7,11 +7,13 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/client-go/util/homedir" "os" + "path" "path/filepath" ) const ( MizuResourcesNamespaceConfigName = "mizu-resources-namespace" + ConfigFilePathCommandName = "config-path" ) type ConfigStruct struct { @@ -27,10 +29,12 @@ type ConfigStruct struct { Telemetry bool `yaml:"telemetry" default:"true"` DumpLogs bool `yaml:"dump-logs" default:"false"` KubeConfigPathStr string `yaml:"kube-config-path"` + ConfigFilePath string `yaml:"config-path,omitempty" readonly:""` } func (config *ConfigStruct) SetDefaults() { config.AgentImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", mizu.Branch, mizu.SemVer) + config.ConfigFilePath = path.Join(mizu.GetMizuFolderPath(), "config.yaml") } func (config *ConfigStruct) ImagePullPolicy() v1.PullPolicy { diff --git a/cli/config/config_test.go b/cli/config/config_test.go index d5028cd94..7f0751f0c 100644 --- a/cli/config/config_test.go +++ b/cli/config/config_test.go @@ -1,6 +1,7 @@ package config_test import ( + "fmt" "github.com/up9inc/mizu/cli/config" "reflect" "strings" @@ -16,7 +17,8 @@ func TestConfigWriteIgnoresReadonlyFields(t *testing.T) { configWithDefaults, _ := config.GetConfigWithDefaults() for _, readonlyField := range readonlyFields { t.Run(readonlyField, func(t *testing.T) { - if strings.Contains(configWithDefaults, readonlyField) { + readonlyFieldToCheck := fmt.Sprintf("\n%s:", readonlyField) + if strings.Contains(configWithDefaults, readonlyFieldToCheck) { t.Errorf("unexpected result - readonly field: %v, config: %v", readonlyField, configWithDefaults) } }) diff --git a/cli/mizu/fsUtils/mizuLogsUtils.go b/cli/mizu/fsUtils/mizuLogsUtils.go index 5edb5d41b..8cfc9524d 100644 --- a/cli/mizu/fsUtils/mizuLogsUtils.go +++ b/cli/mizu/fsUtils/mizuLogsUtils.go @@ -45,10 +45,10 @@ func DumpLogs(provider *kubernetes.Provider, ctx context.Context, filePath strin logger.Log.Debugf("Successfully added log length %d from pod: %s.%s", len(logs), pod.Namespace, pod.Name) } } - if err := AddFileToZip(zipWriter, config.GetConfigFilePath()); err != nil { + if err := AddFileToZip(zipWriter, config.Config.ConfigFilePath); err != nil { logger.Log.Debugf("Failed write file, %v", err) } else { - logger.Log.Debugf("Successfully added file %s", config.GetConfigFilePath()) + logger.Log.Debugf("Successfully added file %s", config.Config.ConfigFilePath) } if err := AddFileToZip(zipWriter, logger.GetLogFilePath()); err != nil { logger.Log.Debugf("Failed write file, %v", err) From a310953f05514d60c9ffb331f3c8554c8a2699af Mon Sep 17 00:00:00 2001 From: Igor Gov Date: Thu, 26 Aug 2021 15:55:05 +0300 Subject: [PATCH 06/11] Fixing call to analysis (#248) --- cli/apiserver/provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/apiserver/provider.go b/cli/apiserver/provider.go index 24f1b193e..e10c7b130 100644 --- a/cli/apiserver/provider.go +++ b/cli/apiserver/provider.go @@ -77,7 +77,7 @@ func (provider *apiServerProvider) RequestAnalysis(analysisDestination string, s if !provider.isReady { return fmt.Errorf("trying to reach api server when not initialized yet") } - urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", provider.url, url.QueryEscape(analysisDestination), sleepIntervalSec) + urlPath := fmt.Sprintf("%s/api/uploadEntries?dest=%s&interval=%v", provider.url, url.QueryEscape(analysisDestination), sleepIntervalSec) u, parseErr := url.ParseRequestURI(urlPath) if parseErr != nil { logger.Log.Fatal("Failed parsing the URL (consider changing the analysis dest URL), err: %v", parseErr) From 80237c809072ce186d61d944059a4c303a1aa5d4 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Mon, 30 Aug 2021 11:43:44 +0300 Subject: [PATCH 07/11] fixed error on invalid config path (#250) --- acceptanceTests/tap_test.go | 13 ++++++++++++- cli/config/config.go | 13 ++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/acceptanceTests/tap_test.go b/acceptanceTests/tap_test.go index 8d921ed1f..e4f8888df 100644 --- a/acceptanceTests/tap_test.go +++ b/acceptanceTests/tap_test.go @@ -71,7 +71,6 @@ func TestTapAndFetch(t *testing.T) { } entries := requestResult.([]interface{}) - if len(entries) == 0 { return fmt.Errorf("unexpected entries result - Expected more than 0 entries") } @@ -523,6 +522,10 @@ func TestTapRedact(t *testing.T) { } entries := requestResult.([]interface{}) + if len(entries) == 0 { + return fmt.Errorf("unexpected entries result - Expected more than 0 entries") + } + firstEntry := entries[0].(map[string]interface{}) entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, firstEntry["id"]) @@ -625,6 +628,10 @@ func TestTapNoRedact(t *testing.T) { } entries := requestResult.([]interface{}) + if len(entries) == 0 { + return fmt.Errorf("unexpected entries result - Expected more than 0 entries") + } + firstEntry := entries[0].(map[string]interface{}) entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, firstEntry["id"]) @@ -727,6 +734,10 @@ func TestTapRegexMasking(t *testing.T) { } entries := requestResult.([]interface{}) + if len(entries) == 0 { + return fmt.Errorf("unexpected entries result - Expected more than 0 entries") + } + firstEntry := entries[0].(map[string]interface{}) entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, firstEntry["id"]) diff --git a/cli/config/config.go b/cli/config/config.go index ab6332b51..5b0032f6f 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -37,11 +37,14 @@ func InitConfig(cmd *cobra.Command) error { return err } - configFilePath := cmd.Flags().Lookup(ConfigFilePathCommandName).Value.String() - + configFilePathFlag := cmd.Flags().Lookup(ConfigFilePathCommandName) + configFilePath := configFilePathFlag.Value.String() if err := mergeConfigFile(configFilePath); err != nil { - return fmt.Errorf("invalid config, %w\n"+ - "you can regenerate the file by removing it (%v) and using `mizu config -r`", err, configFilePath) + _, isPathError := err.(*os.PathError) + if configFilePathFlag.Changed || !isPathError { + return fmt.Errorf("invalid config, %w\n"+ + "you can regenerate the file by removing it (%v) and using `mizu config -r`", err, configFilePath) + } } cmd.Flags().Visit(initFlag) @@ -67,7 +70,7 @@ func GetConfigWithDefaults() (string, error) { func mergeConfigFile(configFilePath string) error { reader, openErr := os.Open(configFilePath) if openErr != nil { - return nil + return openErr } buf, readErr := ioutil.ReadAll(reader) From e25e7925b63a764de1fe9483dfe70273c82e0d5e Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Mon, 30 Aug 2021 15:11:14 +0300 Subject: [PATCH 08/11] fixed version blocking (#251) --- cli/cmd/root.go | 10 +++++++--- cli/config/config.go | 3 +-- cli/mizu/version/versionCheck.go | 7 ++++++- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/cli/cmd/root.go b/cli/cmd/root.go index d14c54acf..39729b6f5 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -10,6 +10,7 @@ import ( "github.com/up9inc/mizu/cli/mizu/fsUtils" "github.com/up9inc/mizu/cli/mizu/version" "github.com/up9inc/mizu/cli/uiUtils" + "time" ) var rootCmd = &cobra.Command{ @@ -34,9 +35,12 @@ func init() { } func printNewVersionIfNeeded(versionChan chan string) { - versionMsg := <-versionChan - if versionMsg != "" { - logger.Log.Infof(uiUtils.Yellow, versionMsg) + select { + case versionMsg := <-versionChan: + if versionMsg != "" { + logger.Log.Infof(uiUtils.Yellow, versionMsg) + } + case <-time.After(2 * time.Second): } } diff --git a/cli/config/config.go b/cli/config/config.go index 5b0032f6f..b75e79cb8 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -40,8 +40,7 @@ func InitConfig(cmd *cobra.Command) error { configFilePathFlag := cmd.Flags().Lookup(ConfigFilePathCommandName) configFilePath := configFilePathFlag.Value.String() if err := mergeConfigFile(configFilePath); err != nil { - _, isPathError := err.(*os.PathError) - if configFilePathFlag.Changed || !isPathError { + if configFilePathFlag.Changed || !os.IsNotExist(err) { return fmt.Errorf("invalid config, %w\n"+ "you can regenerate the file by removing it (%v) and using `mizu config -r`", err, configFilePath) } diff --git a/cli/mizu/version/versionCheck.go b/cli/mizu/version/versionCheck.go index f9f4d2fc5..2c662b2cb 100644 --- a/cli/mizu/version/versionCheck.go +++ b/cli/mizu/version/versionCheck.go @@ -37,6 +37,7 @@ func CheckNewerVersion(versionChan chan string) { latestRelease, _, err := client.Repositories.GetLatestRelease(context.Background(), "up9inc", "mizu") if err != nil { logger.Log.Debugf("[ERROR] Failed to get latest release") + versionChan <- "" return } @@ -49,12 +50,14 @@ func CheckNewerVersion(versionChan chan string) { } if versionFileUrl == "" { logger.Log.Debugf("[ERROR] Version file not found in the latest release") + versionChan <- "" return } res, err := http.Get(versionFileUrl) if err != nil { logger.Log.Debugf("[ERROR] Failed to get the version file %v", err) + versionChan <- "" return } @@ -62,6 +65,7 @@ func CheckNewerVersion(versionChan chan string) { res.Body.Close() if err != nil { logger.Log.Debugf("[ERROR] Failed to read the version file -> %v", err) + versionChan <- "" return } gitHubVersion := string(data) @@ -73,6 +77,7 @@ func CheckNewerVersion(versionChan chan string) { if gitHubVersionSemVer.GreaterThan(currentSemVer) { versionChan <- fmt.Sprintf("Update available! %v -> %v (%v)", mizu.SemVer, gitHubVersion, *latestRelease.HTMLURL) + } else { + versionChan <- "" } - versionChan <- "" } From ab7c4e72c6129b32b29afc84427c79a320964fc9 Mon Sep 17 00:00:00 2001 From: gadotroee <55343099+gadotroee@users.noreply.github.com> Date: Tue, 31 Aug 2021 15:27:13 +0300 Subject: [PATCH 09/11] no message (#253) --- agent/main.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/agent/main.go b/agent/main.go index 8f0ffd4d6..8aff329c3 100644 --- a/agent/main.go +++ b/agent/main.go @@ -107,6 +107,7 @@ func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) { SocketHarOutChannel: socketHarOutputChannel, } + app.Use(DisableRootStaticCache()) app.Use(static.ServeRoot("/", "./site")) app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before @@ -119,6 +120,17 @@ func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) { utils.StartServer(app) } +func DisableRootStaticCache() gin.HandlerFunc { + return func(c *gin.Context) { + if c.Request.RequestURI == "/" { + // Disable cache only for the main static route + c.Writer.Header().Set("Cache-Control", "no-store") + } + + c.Next() + } +} + func CORSMiddleware() gin.HandlerFunc { return func(c *gin.Context) { c.Writer.Header().Set("Access-Control-Allow-Origin", "*") From 3644fdb5335310ba58eb5aeabb4b60e557dac801 Mon Sep 17 00:00:00 2001 From: Neim Elezi <49072837+imceZZ@users.noreply.github.com> Date: Wed, 1 Sep 2021 12:39:02 +0200 Subject: [PATCH 10/11] Feature/tra 3533 ssl connection pop up (#223) * pop-up message for HTTPS domains is modified * scroll added on hover of the TLS pop-up * domains that were for testing are removed * height of the pop-up is decreased * condition for return is changed --- ui/src/App.sass | 15 ++++++++++++++- ui/src/App.tsx | 7 ++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/ui/src/App.sass b/ui/src/App.sass index 0b409a236..7ef640155 100644 --- a/ui/src/App.sass +++ b/ui/src/App.sass @@ -22,4 +22,17 @@ margin-left: 10px font-size: 11px font-weight: bold - color: $light-blue-color \ No newline at end of file + color: $light-blue-color + + .httpsDomains + display: none + margin: 0 + padding: 0 + list-style: none + + .customWarningStyle + &:hover + overflow-y: scroll + height: 85px + .httpsDomains + display: block diff --git a/ui/src/App.tsx b/ui/src/App.tsx index fa5b8153c..3ea758f8d 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -92,7 +92,7 @@ const App = () => { - + return (
@@ -119,8 +119,9 @@ const App = () => {
- setUserDismissedTLSWarning(true)} severity="warning"> - Mizu is detecting TLS traffic{addressesWithTLS.size ? ` (directed to ${Array.from(addressesWithTLS).join(", ")})` : ''}, this type of traffic will not be displayed. + setUserDismissedTLSWarning(true)} severity="warning"> + Mizu is detecting TLS traffic, this type of traffic will not be displayed. + {addressesWithTLS.size > 0 &&
    {Array.from(addressesWithTLS, address =>
  • {address}
    • )}
}
From 17fa163ee310d94c3c6efea0509b014312290842 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Wed, 1 Sep 2021 15:30:37 +0300 Subject: [PATCH 11/11] added proxy logs, added events logs (#254) --- cli/apiserver/provider.go | 20 +++++++++++++++----- cli/cmd/common.go | 2 ++ cli/cmd/fetch.go | 2 +- cli/cmd/fetchRunner.go | 2 +- cli/cmd/tapRunner.go | 2 +- cli/cmd/viewRunner.go | 2 +- cli/config/envConfig.go | 24 ++++++++++++++++++++++++ cli/kubernetes/provider.go | 10 ++++++++++ cli/kubernetes/proxy.go | 1 + cli/mizu/fsUtils/mizuLogsUtils.go | 17 +++++++++++++++++ 10 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 cli/config/envConfig.go diff --git a/cli/apiserver/provider.go b/cli/apiserver/provider.go index e10c7b130..5b8ca44cf 100644 --- a/cli/apiserver/provider.go +++ b/cli/apiserver/provider.go @@ -5,6 +5,7 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/up9inc/mizu/cli/config" "github.com/up9inc/mizu/cli/logger" "github.com/up9inc/mizu/cli/uiUtils" "github.com/up9inc/mizu/shared" @@ -18,18 +19,27 @@ import ( type apiServerProvider struct { url string isReady bool + retries int } -var Provider = apiServerProvider{} +var Provider = apiServerProvider{retries: config.GetIntEnvConfig(config.ApiServerRetries, 20)} -func (provider *apiServerProvider) InitAndTestConnection(url string, retries int) error { +func (provider *apiServerProvider) InitAndTestConnection(url string) error { healthUrl := fmt.Sprintf("%s/", url) - retriesLeft := retries + retriesLeft := provider.retries for retriesLeft > 0 { if response, err := http.Get(healthUrl); err != nil { logger.Log.Debugf("[ERROR] failed connecting to api server %v", err) } else if response.StatusCode != 200 { - logger.Log.Debugf("can't connect to api server yet, response status code %v", response.StatusCode) + responseBody := "" + data, readErr := ioutil.ReadAll(response.Body) + if readErr == nil { + responseBody = string(data) + } + + logger.Log.Debugf("can't connect to api server yet, response status code: %v, body: %v", response.StatusCode, responseBody) + + response.Body.Close() } else { logger.Log.Debugf("connection test to api server passed successfully") break @@ -40,7 +50,7 @@ func (provider *apiServerProvider) InitAndTestConnection(url string, retries int if retriesLeft == 0 { provider.isReady = false - return fmt.Errorf("couldn't reach the api server after %v retries", retries) + return fmt.Errorf("couldn't reach the api server after %v retries", provider.retries) } provider.url = url provider.isReady = true diff --git a/cli/cmd/common.go b/cli/cmd/common.go index 907663b2c..1f09c6ace 100644 --- a/cli/cmd/common.go +++ b/cli/cmd/common.go @@ -26,6 +26,8 @@ func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel "Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName)) cancel() } + + logger.Log.Debugf("proxy ended") } func waitForFinish(ctx context.Context, cancel context.CancelFunc) { diff --git a/cli/cmd/fetch.go b/cli/cmd/fetch.go index dfddc350d..15d124726 100644 --- a/cli/cmd/fetch.go +++ b/cli/cmd/fetch.go @@ -18,7 +18,7 @@ var fetchCmd = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { go telemetry.ReportRun("fetch", config.Config.Fetch) - if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl(), 1); err != nil { + if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl()); err != nil { logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, make sure one running") return nil } diff --git a/cli/cmd/fetchRunner.go b/cli/cmd/fetchRunner.go index aeff496af..0b539244a 100644 --- a/cli/cmd/fetchRunner.go +++ b/cli/cmd/fetchRunner.go @@ -9,7 +9,7 @@ import ( ) func RunMizuFetch() { - if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl(), 5); err != nil { + if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl()); err != nil { logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, check logs") } diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 5c44b9348..b1ce78cb8 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -497,7 +497,7 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi isPodReady = true go startProxyReportErrorIfAny(kubernetesProvider, cancel) - if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl(), 20); err != nil { + if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl()); err != nil { logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, check logs") cancel() break diff --git a/cli/cmd/viewRunner.go b/cli/cmd/viewRunner.go index 4ef1d6dfa..69bd2a27c 100644 --- a/cli/cmd/viewRunner.go +++ b/cli/cmd/viewRunner.go @@ -43,7 +43,7 @@ func runMizuView() { logger.Log.Infof("Establishing connection to k8s cluster...") go startProxyReportErrorIfAny(kubernetesProvider, cancel) - if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl(), 10); err != nil { + if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl()); err != nil { logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, check logs") return } diff --git a/cli/config/envConfig.go b/cli/config/envConfig.go new file mode 100644 index 000000000..78f91e00b --- /dev/null +++ b/cli/config/envConfig.go @@ -0,0 +1,24 @@ +package config + +import ( + "os" + "strconv" +) + +const ( + ApiServerRetries = "API_SERVER_RETRIES" +) + +func GetIntEnvConfig(key string, defaultValue int) int { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + + intValue, err := strconv.Atoi(value) + if err != nil { + return defaultValue + } + + return intValue +} diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index 1603b1307..486bf1377 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -739,6 +739,16 @@ func (provider *Provider) GetPodLogs(namespace string, podName string, ctx conte return str, nil } +func (provider *Provider) GetNamespaceEvents(namespace string, ctx context.Context) (string, error) { + eventsOpts := metav1.ListOptions{} + eventList, err := provider.clientSet.CoreV1().Events(namespace).List(ctx, eventsOpts) + if err != nil { + return "", fmt.Errorf("error getting events on ns: %s, %w", namespace, err) + } + + return eventList.String(), nil +} + func getClientSet(config *restclient.Config) (*kubernetes.Clientset, error) { clientSet, err := kubernetes.NewForConfig(config) if err != nil { diff --git a/cli/kubernetes/proxy.go b/cli/kubernetes/proxy.go index 44397bdc4..8489f0b4a 100644 --- a/cli/kubernetes/proxy.go +++ b/cli/kubernetes/proxy.go @@ -39,6 +39,7 @@ func StartProxy(kubernetesProvider *Provider, mizuPort uint16, mizuNamespace str server := http.Server{ Handler: mux, } + return server.Serve(l) } diff --git a/cli/mizu/fsUtils/mizuLogsUtils.go b/cli/mizu/fsUtils/mizuLogsUtils.go index 8cfc9524d..54ab05d28 100644 --- a/cli/mizu/fsUtils/mizuLogsUtils.go +++ b/cli/mizu/fsUtils/mizuLogsUtils.go @@ -39,22 +39,39 @@ func DumpLogs(provider *kubernetes.Provider, ctx context.Context, filePath strin } else { logger.Log.Debugf("Successfully read log length %d for pod: %s.%s", len(logs), pod.Namespace, pod.Name) } + if err := AddStrToZip(zipWriter, logs, fmt.Sprintf("%s.%s.log", pod.Namespace, pod.Name)); err != nil { logger.Log.Errorf("Failed write logs, %v", err) } else { logger.Log.Debugf("Successfully added log length %d from pod: %s.%s", len(logs), pod.Namespace, pod.Name) } } + + events, err := provider.GetNamespaceEvents(config.Config.MizuResourcesNamespace, ctx) + if err != nil { + logger.Log.Debugf("Failed to get k8b events, %v", err) + } else { + logger.Log.Debugf("Successfully read events for k8b namespace: %s", config.Config.MizuResourcesNamespace) + } + + if err := AddStrToZip(zipWriter, events, fmt.Sprintf("%s_events.log", config.Config.MizuResourcesNamespace)); err != nil { + logger.Log.Debugf("Failed write logs, %v", err) + } else { + logger.Log.Debugf("Successfully added events for k8b namespace: %s", config.Config.MizuResourcesNamespace) + } + if err := AddFileToZip(zipWriter, config.Config.ConfigFilePath); err != nil { logger.Log.Debugf("Failed write file, %v", err) } else { logger.Log.Debugf("Successfully added file %s", config.Config.ConfigFilePath) } + if err := AddFileToZip(zipWriter, logger.GetLogFilePath()); err != nil { logger.Log.Debugf("Failed write file, %v", err) } else { logger.Log.Debugf("Successfully added file %s", logger.GetLogFilePath()) } + logger.Log.Infof("You can find the zip file with all logs in %s\n", filePath) return nil }