From 31dcfc4b2e689dc6f532c8a0d719e7c955d7be5d Mon Sep 17 00:00:00 2001 From: gadotroee <55343099+gadotroee@users.noreply.github.com> Date: Tue, 8 Jun 2021 11:17:02 +0300 Subject: [PATCH] TRA-3318 - Cookies not null and fix har file names (#69) * no message --- api/pkg/controllers/entries_controller.go | 5 +- .../messageSensitiveDataCleaner.go | 4 +- cli/cmd/tap.go | 2 + cli/cmd/tapRunner.go | 59 +++++++++++-------- cli/cmd/viewRunner.go | 2 +- cli/kubernetes/provider.go | 30 +++------- cli/mizu/consts.go | 5 +- 7 files changed, 54 insertions(+), 53 deletions(-) diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index 293f02c29..06b96b5b8 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -94,12 +94,13 @@ func GetHARs(c *fiber.Ctx) error { _ = json.Unmarshal([]byte(entryData.Entry), &harEntry) sourceOfEntry := entryData.ResolvedSource - if harOfSource, ok := harsObject[sourceOfEntry]; ok { + fileName := fmt.Sprintf("%s.har", sourceOfEntry) + if harOfSource, ok := harsObject[fileName]; ok { harOfSource.Log.Entries = append(harOfSource.Log.Entries, &harEntry) } else { var entriesHar []*har.Entry entriesHar = append(entriesHar, &harEntry) - harsObject[sourceOfEntry] = &models.ExtendedHAR{ + harsObject[fileName] = &models.ExtendedHAR{ Log: &models.ExtendedLog{ Version: "1.2", Creator: &models.ExtendedCreator{ diff --git a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go index 7cf9660f9..f302660f2 100644 --- a/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go +++ b/api/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go @@ -18,8 +18,8 @@ func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, opt filterHarHeaders(harOutputItem.HarEntry.Request.Headers) filterHarHeaders(harOutputItem.HarEntry.Response.Headers) - harOutputItem.HarEntry.Request.Cookies = nil - harOutputItem.HarEntry.Response.Cookies = nil + harOutputItem.HarEntry.Request.Cookies = make([]har.Cookie, 0, 0) + harOutputItem.HarEntry.Response.Cookies = make([]har.Cookie, 0, 0) harOutputItem.HarEntry.Request.URL = filterUrl(harOutputItem.HarEntry.Request.URL) for i, queryString := range harOutputItem.HarEntry.Request.QueryString { diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 5c795d935..69c4b988a 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -12,6 +12,7 @@ import ( type MizuTapOptions struct { GuiPort uint16 Namespace string + AllNamespaces bool KubeConfigPath string MizuImage string MizuPodPort uint16 @@ -48,6 +49,7 @@ func init() { tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver") tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector") + tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces") tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file") tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector") tapCmd.Flags().Uint16VarP(&mizuTapOptions.MizuPodPort, "mizu-port", "", 8899, "Port which mizu cli will attempt to forward from the mizu collector pod") diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 1fb81edb4..e01294e1d 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -31,7 +31,8 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { if err != nil { return } - kubernetesProvider := kubernetes.NewProvider(tappingOptions.KubeConfigPath, tappingOptions.Namespace) + + kubernetesProvider := kubernetes.NewProvider(tappingOptions.KubeConfigPath) defer cleanUpMizuResources(kubernetesProvider) ctx, cancel := context.WithCancel(context.Background()) @@ -43,7 +44,7 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) { currentlyTappedPods = matchingPods } - nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(ctx, kubernetesProvider, currentlyTappedPods) + nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods) if err != nil { return } @@ -132,20 +133,20 @@ func createMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) { fmt.Printf("\nRemoving mizu resources\n") - removalCtx, _ := context.WithTimeout(context.Background(), 5 * time.Second) + removalCtx, _ := context.WithTimeout(context.Background(), 5*time.Second) if err := kubernetesProvider.RemovePod(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil { - fmt.Printf("Error removing Pod %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err); + fmt.Printf("Error removing Pod %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err) } if err := kubernetesProvider.RemoveService(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil { - fmt.Printf("Error removing Service %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err); + fmt.Printf("Error removing Service %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err) } if err := kubernetesProvider.RemoveDaemonSet(removalCtx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil { - fmt.Printf("Error removing DaemonSet %s in namespace %s: %s (%v,%+v)\n", mizu.TapperDaemonSetName, mizu.ResourcesNamespace, err, err, err); + fmt.Printf("Error removing DaemonSet %s in namespace %s: %s (%v,%+v)\n", mizu.TapperDaemonSetName, mizu.ResourcesNamespace, err, err, err) } } func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, podRegex *regexp.Regexp, tappingOptions *MizuTapOptions) { - added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, kubernetesProvider.Namespace), podRegex) + added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, getNamespace(tappingOptions, kubernetesProvider)), podRegex) restartTappers := func() { if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegex); err != nil { @@ -155,7 +156,7 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro currentlyTappedPods = matchingPods } - nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(ctx, kubernetesProvider, currentlyTappedPods) + nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods) if err != nil { fmt.Printf("Error building node to ips map: %s (%v,%+v)\n", err, err, err) cancel() @@ -170,14 +171,14 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro for { select { - case newTarget := <- added: + case newTarget := <-added: fmt.Printf("+%s\n", newTarget.Name) - case removedTarget := <- removed: + case removedTarget := <-removed: fmt.Printf("-%s\n", removedTarget.Name) restartTappersDebouncer.SetOn() - case modifiedTarget := <- modified: + case modifiedTarget := <-modified: // Act only if the modified pod has already obtained an IP address. // After filtering for IPs, on a normal pod restart this includes the following events: // - Pod deletion @@ -188,11 +189,11 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro restartTappersDebouncer.SetOn() } - case <- errorChan: + case <-errorChan: // TODO: Does this also perform cleanup? cancel() - case <- ctx.Done(): + case <-ctx.Done(): return } } @@ -205,13 +206,13 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi var portForward *kubernetes.PortForward for { select { - case <- added: + case <-added: continue - case <- removed: + case <-removed: fmt.Printf("%s removed\n", mizu.AggregatorPodName) cancel() return - case modifiedPod := <- modified: + case modifiedPod := <-modified: if modifiedPod.Status.Phase == "Running" && !isPodReady { isPodReady = true var err error @@ -223,16 +224,16 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi } } - case <- time.After(25 * time.Second): + case <-time.After(25 * time.Second): if !isPodReady { fmt.Printf("error: %s pod was not ready in time", mizu.AggregatorPodName) cancel() } - case <- errorChan: + case <-errorChan: cancel() - case <- ctx.Done(): + case <-ctx.Done(): if portForward != nil { portForward.Stop() } @@ -261,12 +262,12 @@ func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.P return true } -func getNodeHostToTappedPodIpsMap(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappedPods []core.Pod) (map[string][]string, error) { +func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) (map[string][]string, error) { nodeToTappedPodIPMap := make(map[string][]string, 0) for _, pod := range tappedPods { existingList := nodeToTappedPodIPMap[pod.Spec.NodeName] if existingList == nil { - nodeToTappedPodIPMap[pod.Spec.NodeName] = []string {pod.Status.PodIP} + nodeToTappedPodIPMap[pod.Spec.NodeName] = []string{pod.Status.PodIP} } else { nodeToTappedPodIPMap[pod.Spec.NodeName] = append(nodeToTappedPodIPMap[pod.Spec.NodeName], pod.Status.PodIP) } @@ -280,9 +281,9 @@ func waitForFinish(ctx context.Context, cancel context.CancelFunc) { // block until ctx cancel is called or termination signal is received select { - case <- ctx.Done(): + case <-ctx.Done(): break - case <- sigChan: + case <-sigChan: cancel() } } @@ -296,7 +297,7 @@ func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOption for { select { - case <- ctx.Done(): + case <-ctx.Done(): return default: err = controlSocket.SendNewTappedPodsListMessage(currentlyTappedPods) @@ -308,3 +309,13 @@ func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOption } } + +func getNamespace(tappingOptions *MizuTapOptions, kubernetesProvider *kubernetes.Provider) string { + if tappingOptions.AllNamespaces { + return mizu.K8sAllNamespaces + } else if len(tappingOptions.Namespace) > 0 { + return tappingOptions.Namespace + } else { + return kubernetesProvider.CurrentNamespace() + } +} diff --git a/cli/cmd/viewRunner.go b/cli/cmd/viewRunner.go index 18e2bf9e0..059a9b2e2 100644 --- a/cli/cmd/viewRunner.go +++ b/cli/cmd/viewRunner.go @@ -9,7 +9,7 @@ import ( ) func runMizuView() { - kubernetesProvider := kubernetes.NewProvider("", "") + kubernetesProvider := kubernetes.NewProvider("") ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index 58d96b18c..f7d7e68f7 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/up9inc/mizu/cli/mizu" "path/filepath" "regexp" @@ -42,7 +43,7 @@ const ( fieldManagerName = "mizu-manager" ) -func NewProvider(kubeConfigPath string, overrideNamespace string) *Provider { +func NewProvider(kubeConfigPath string) *Provider { kubernetesConfig := loadKubernetesConfiguration(kubeConfigPath) restClientConfig, err := kubernetesConfig.ClientConfig() if err != nil { @@ -50,25 +51,18 @@ func NewProvider(kubeConfigPath string, overrideNamespace string) *Provider { } clientSet := getClientSet(restClientConfig) - var namespace string - if len(overrideNamespace) > 0 { - namespace = overrideNamespace - } else { - configuredNamespace, _, err := kubernetesConfig.Namespace() - if err != nil { - panic(err) - } - namespace = configuredNamespace - } - return &Provider{ clientSet: clientSet, kubernetesConfig: kubernetesConfig, clientConfig: *restClientConfig, - Namespace: namespace, } } +func (provider *Provider) CurrentNamespace() string { + ns, _, _ := provider.kubernetesConfig.Namespace() + return ns +} + func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) watch.Interface { watcher, err := provider.clientSet.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{Watch: true}) if err != nil { @@ -77,14 +71,6 @@ func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) w return watcher } -func (provider *Provider) GetPods(ctx context.Context, namespace string) { - pods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) - if err != nil { - panic(err.Error()) - } - fmt.Printf("There are %d pods in Namespace %s\n", len(pods.Items), namespace) -} - func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace string, podName string, podImage string, linkServiceAccount bool, mizuApiFilteringOptions *shared.TrafficFilteringOptions) (*core.Pod, error) { marshaledFilteringOptions, err := json.Marshal(mizuApiFilteringOptions) if err != nil { @@ -316,7 +302,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac } func (provider *Provider) GetAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp) ([]core.Pod, error) { - pods, err := provider.clientSet.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) + pods, err := provider.clientSet.CoreV1().Pods(mizu.K8sAllNamespaces).List(ctx, metav1.ListOptions{}) if err != nil { return nil, err } diff --git a/cli/mizu/consts.go b/cli/mizu/consts.go index 22532d0e6..59d33f1b0 100644 --- a/cli/mizu/consts.go +++ b/cli/mizu/consts.go @@ -1,8 +1,8 @@ package mizu var ( - Version = "v0.0.1" - Branch = "develop" + Version = "v0.0.1" + Branch = "develop" GitCommitHash = "" // this var is overridden using ldflags in makefile when building ) @@ -11,4 +11,5 @@ const ( TapperDaemonSetName = "mizu-tapper-daemon-set" AggregatorPodName = "mizu-collector" TapperPodName = "mizu-tapper" + K8sAllNamespaces = "" )