diff --git a/Makefile b/Makefile index 6aa9b0847..bb85ff8ea 100644 --- a/Makefile +++ b/Makefile @@ -33,13 +33,8 @@ agent: ## Build agent. @(cd agent; go build -o build/mizuagent main.go) @ls -l agent/build -#tap: ## build tap binary -# @(cd tap; go build -o build/tap ./src) -# @ls -l tap/build - -docker: ## Build Docker image. - @(echo "building docker image" ) - ./build-push-featurebranch.sh +docker: ## Build and publish agent docker image. + $(MAKE) push-docker push: push-docker push-cli ## Build and publish agent docker image & CLI. diff --git a/README.md b/README.md index 34d720366..e09ab9988 100644 --- a/README.md +++ b/README.md @@ -50,12 +50,14 @@ Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page. - list - watch - create + - delete - apiGroups: - "" resources: - services verbs: - create + - delete - apiGroups: - apps resources: @@ -63,11 +65,13 @@ Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page. verbs: - create - patch + - delete - apiGroups: - "" resources: - namespaces verbs: + - get - list - watch - create @@ -79,7 +83,8 @@ Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page. verbs: - get ``` -3. Optionally, for resolving traffic ip to kubernetes service name, mizu needs below permissions + +3. Optionally, for resolving traffic IP to kubernetes service name, mizu needs below permissions ```yaml - apiGroups: @@ -88,6 +93,10 @@ Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page. - pods verbs: - get + - list + - watch + - create + - delete - apiGroups: - "" resources: @@ -96,6 +105,72 @@ Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page. - get - list - watch + - create + - delete +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - create + - patch + - delete +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - "" + resources: + - services/proxy + verbs: + - get +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - get + - create + - delete +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + verbs: + - get + - create + - delete +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + verbs: + - get + - create + - delete +- apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + verbs: + - get + - create + - delete +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + verbs: + - get + - create + - delete - apiGroups: - apps - extensions @@ -124,6 +199,97 @@ Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page. - get - list - watch +``` + +4. Optionally, in order to use the policy rules validation feature, mizu requires the following additional permissions: + +```yaml +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - create + - delete +``` + +5. Alternatively, in order to restrict mizu to one namespace only (by setting `agent.namespace` in the config file), mizu needs the following permissions in that namespace: + +```yaml +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - "" + resources: + - services + verbs: + - get + - create + - delete +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - get + - create + - patch + - delete +- apiGroups: + - "" + resources: + - services/proxy + verbs: + - get +``` + +6. To restrict mizu to one namespace while also resolving IPs, mizu needs the following permissions in that namespace: + +```yaml +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - get + - create + - patch + - delete +- apiGroups: + - "" + resources: + - services/proxy + verbs: + - get - apiGroups: - "" resources: @@ -131,22 +297,51 @@ Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page. verbs: - get - create + - delete - apiGroups: - rbac.authorization.k8s.io resources: - - clusterroles + - roles verbs: - - list + - get - create - delete - apiGroups: - rbac.authorization.k8s.io resources: - - clusterrolebindings + - rolebindings verbs: - - list + - get - create - delete +- apiGroups: + - apps + - extensions + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - apps + - extensions + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - "" + - apps + - extensions + resources: + - endpoints + verbs: + - get + - list + - watch ``` See `examples/roles` for example `clusterroles`. diff --git a/agent/main.go b/agent/main.go index 2d7c8096e..381433832 100644 --- a/agent/main.go +++ b/agent/main.go @@ -21,21 +21,24 @@ import ( "strings" ) -var shouldTap = flag.Bool("tap", false, "Run in tapper mode without API") -var apiServer = flag.Bool("api-server", false, "Run in API server mode with API") -var standalone = flag.Bool("standalone", false, "Run in standalone tapper and API mode") +var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API") +var apiServerMode = flag.Bool("api-server", false, "Run in API server mode with API") +var standaloneMode = flag.Bool("standalone", false, "Run in standalone tapper and API mode") var apiServerAddress = flag.String("api-server-address", "", "Address of mizu API server") +var namespace = flag.String("namespace", "", "Resolve IPs if they belong to resources in this namespace (default is all)") func main() { flag.Parse() hostMode := os.Getenv(shared.HostModeEnvVar) == "1" tapOpts := &tap.TapOpts{HostMode: hostMode} - if !*shouldTap && !*apiServer && !*standalone { + if !*tapperMode && !*apiServerMode && !*standaloneMode { panic("One of the flags --tap, --api or --standalone must be provided") } - if *standalone { + if *standaloneMode { + api.StartResolving(*namespace) + harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts) filteredHarChannel := make(chan *tap.OutputChannelItem) @@ -44,7 +47,7 @@ func main() { go api.StartReadingOutbound(outboundLinkOutputChannel) hostApi(nil) - } else if *shouldTap { + } else if *tapperMode { if *apiServerAddress == "" { panic("API server address must be provided with --api-server-address when using --tap") } @@ -64,7 +67,9 @@ func main() { go pipeTapChannelToSocket(socketConnection, harOutputChannel) go pipeOutboundLinksChannelToSocket(socketConnection, outboundLinkOutputChannel) - } else if *apiServer { + } else if *apiServerMode { + api.StartResolving(*namespace) + socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000) filteredHarChannel := make(chan *tap.OutputChannelItem) diff --git a/agent/pkg/api/main.go b/agent/pkg/api/main.go index 70439fc47..5c41251fa 100644 --- a/agent/pkg/api/main.go +++ b/agent/pkg/api/main.go @@ -26,7 +26,7 @@ import ( var k8sResolver *resolver.Resolver -func init() { +func StartResolving(namespace string) { errOut := make(chan error, 100) res, err := resolver.NewFromInCluster(errOut) if err != nil { @@ -34,7 +34,7 @@ func init() { return } ctx := context.Background() - res.Start(ctx) + res.Start(ctx, namespace) go func() { for { select { diff --git a/agent/pkg/resolver/resolver.go b/agent/pkg/resolver/resolver.go index e2b09e4c3..6b8bd6e9f 100644 --- a/agent/pkg/resolver/resolver.go +++ b/agent/pkg/resolver/resolver.go @@ -18,17 +18,20 @@ const ( ) type Resolver struct { - clientConfig *restclient.Config - clientSet *kubernetes.Clientset - nameMap map[string]string - serviceMap map[string]string - isStarted bool - errOut chan error + clientConfig *restclient.Config + clientSet *kubernetes.Clientset + nameMap map[string]string + serviceMap map[string]string + isStarted bool + errOut chan error + namespace string } -func (resolver *Resolver) Start(ctx context.Context) { +func (resolver *Resolver) Start(ctx context.Context, namespace string) { if !resolver.isStarted { resolver.isStarted = true + resolver.namespace = namespace + go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchServices) go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchEndpoints) go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchPods) @@ -54,7 +57,7 @@ func (resolver *Resolver) CheckIsServiceIP(address string) bool { func (resolver *Resolver) watchPods(ctx context.Context) error { // empty namespace makes the client watch all namespaces - watcher, err := resolver.clientSet.CoreV1().Pods("").Watch(ctx, metav1.ListOptions{Watch: true}) + watcher, err := resolver.clientSet.CoreV1().Pods(resolver.namespace).Watch(ctx, metav1.ListOptions{Watch: true}) if err != nil { return err } @@ -77,7 +80,7 @@ func (resolver *Resolver) watchPods(ctx context.Context) error { func (resolver *Resolver) watchEndpoints(ctx context.Context) error { // empty namespace makes the client watch all namespaces - watcher, err := resolver.clientSet.CoreV1().Endpoints("").Watch(ctx, metav1.ListOptions{Watch: true}) + watcher, err := resolver.clientSet.CoreV1().Endpoints(resolver.namespace).Watch(ctx, metav1.ListOptions{Watch: true}) if err != nil { return err } @@ -120,7 +123,7 @@ func (resolver *Resolver) watchEndpoints(ctx context.Context) error { func (resolver *Resolver) watchServices(ctx context.Context) error { // empty namespace makes the client watch all namespaces - watcher, err := resolver.clientSet.CoreV1().Services("").Watch(ctx, metav1.ListOptions{Watch: true}) + watcher, err := resolver.clientSet.CoreV1().Services(resolver.namespace).Watch(ctx, metav1.ListOptions{Watch: true}) if err != nil { return err } diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 098a54824..bfb62dcc6 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -6,6 +6,7 @@ import ( "github.com/creasty/defaults" "github.com/spf13/cobra" + "github.com/up9inc/mizu/cli/errormessage" "github.com/up9inc/mizu/cli/mizu" "github.com/up9inc/mizu/cli/mizu/configStructs" "github.com/up9inc/mizu/cli/uiUtils" @@ -31,7 +32,7 @@ Supported protocols are HTTP and gRPC.`, } if err := mizu.Config.Tap.Validate(); err != nil { - return err + return errormessage.FormatError(err) } mizu.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", mizu.Config.Tap.HumanMaxEntriesDBSize) diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index f3d8b0228..2a3868c1c 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -13,6 +13,7 @@ import ( "syscall" "time" + "github.com/up9inc/mizu/cli/errormessage" "github.com/up9inc/mizu/cli/kubernetes" "github.com/up9inc/mizu/cli/mizu" "github.com/up9inc/mizu/cli/uiUtils" @@ -20,31 +21,35 @@ import ( "github.com/up9inc/mizu/shared/debounce" yaml "gopkg.in/yaml.v3" core "k8s.io/api/core/v1" - errors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/clientcmd" ) -var mizuServiceAccountExists bool -var apiServerService *core.Service - const ( - updateTappersDelay = 5 * time.Second cleanupTimeout = time.Minute + updateTappersDelay = 5 * time.Second ) -var currentlyTappedPods []core.Pod +type tapState struct { + apiServerService *core.Service + currentlyTappedPods []core.Pod + mizuServiceAccountExists bool + doNotRemoveConfigMap bool +} + +var state tapState func RunMizuTap() { mizuApiFilteringOptions, err := getMizuApiFilteringOptions() if err != nil { + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error parsing regex-masking: %v", errormessage.FormatError(err))) return } var mizuValidationRules string if mizu.Config.Tap.EnforcePolicyFile != "" { mizuValidationRules, err = readValidationRules(mizu.Config.Tap.EnforcePolicyFile) if err != nil { - mizu.Log.Infof("error: %v", err) + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading policy file: %v", errormessage.FormatError(err))) return } } @@ -52,11 +57,11 @@ func RunMizuTap() { kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.Tap.KubeConfigPath) if err != nil { if clientcmd.IsEmptyConfig(err) { - mizu.Log.Infof(uiUtils.Red, "Couldn't find the kube config file, or file is empty. Try adding '--kube-config='\n") + mizu.Log.Errorf(uiUtils.Error, "Couldn't find the kube config file, or file is empty. Try adding '--kube-config='\n") return } if clientcmd.IsConfigurationInvalid(err) { - mizu.Log.Infof(uiUtils.Red, "Invalid kube config file. Try using a different config with '--kube-config='\n") + mizu.Log.Errorf(uiUtils.Error, "Invalid kube config file. Try using a different config with '--kube-config='\n") return } } @@ -76,28 +81,26 @@ func RunMizuTap() { mizu.Log.Infof("Tapping pods in %s", namespacesStr) if err, _ := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespace); err != nil { - mizu.Log.Infof("Error listing pods: %v", err) + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error getting pods by regex: %v", errormessage.FormatError(err))) return } - if len(currentlyTappedPods) == 0 { + if len(state.currentlyTappedPods) == 0 { var suggestionStr string if targetNamespace != mizu.K8sAllNamespaces { - suggestionStr = "\nSelect a different namespace with -n or tap all namespaces with -A" + suggestionStr = ". Select a different namespace with -n or tap all namespaces with -A" } - mizu.Log.Infof("Did not find any pods matching the regex argument%s", suggestionStr) + mizu.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any pods matching the regex argument%s", suggestionStr)) } if mizu.Config.Tap.DryRun { return } - nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods) - if err != nil { - return - } + nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods) if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions, mizuValidationRules); err != nil { + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err))) return } @@ -118,8 +121,10 @@ func readValidationRules(file string) (string, error) { } func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, mizuApiFilteringOptions *shared.TrafficFilteringOptions, mizuValidationRules string) error { - if err := createMizuNamespace(ctx, kubernetesProvider); err != nil { - return err + if mizu.Config.IsOwnNamespace() { + if err := createMizuNamespace(ctx, kubernetesProvider); err != nil { + return err + } } if err := createMizuApiServer(ctx, kubernetesProvider, mizuApiFilteringOptions); err != nil { @@ -131,50 +136,57 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro } if err := createMizuConfigmap(ctx, kubernetesProvider, mizuValidationRules); err != nil { - return err + mizu.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err))) + state.doNotRemoveConfigMap = true + } else if mizuValidationRules == "" { + state.doNotRemoveConfigMap = true } return nil } func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, data string) error { - err := kubernetesProvider.ApplyConfigMap(ctx, mizu.ResourcesNamespace, mizu.ConfigMapName, data) - if err != nil { - fmt.Printf("Error creating mizu configmap: %v\n", err) - } - return nil + err := kubernetesProvider.CreateConfigMap(ctx, mizu.Config.ResourcesNamespace(), mizu.ConfigMapName, data) + return err } func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Provider) error { - _, err := kubernetesProvider.CreateNamespace(ctx, mizu.ResourcesNamespace) - if err != nil { - mizu.Log.Infof("Error creating Namespace %s: %v", mizu.ResourcesNamespace, err) - return err - } - mizu.Log.Debugf("Successfully creating Namespace %s", mizu.ResourcesNamespace) - return nil + _, err := kubernetesProvider.CreateNamespace(ctx, mizu.Config.ResourcesNamespace()) + return err } func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error { var err error - mizuServiceAccountExists = createRBACIfNecessary(ctx, kubernetesProvider) + state.mizuServiceAccountExists, err = createRBACIfNecessary(ctx, kubernetesProvider) + if err != nil { + mizu.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err))) + } + var serviceAccountName string - if mizuServiceAccountExists { + if state.mizuServiceAccountExists { serviceAccountName = mizu.ServiceAccountName } else { serviceAccountName = "" } - _, err = kubernetesProvider.CreateMizuApiServerPod(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName, mizu.Config.MizuImage, serviceAccountName, mizuApiFilteringOptions, mizu.Config.Tap.MaxEntriesDBSizeBytes()) + + opts := &kubernetes.ApiServerOptions{ + Namespace: mizu.Config.ResourcesNamespace(), + PodName: mizu.ApiServerPodName, + PodImage: mizu.Config.MizuImage, + ServiceAccountName: serviceAccountName, + IsNamespaceRestricted: !mizu.Config.IsOwnNamespace(), + MizuApiFilteringOptions: mizuApiFilteringOptions, + MaxEntriesDBSizeBytes: mizu.Config.Tap.MaxEntriesDBSizeBytes(), + } + _, err = kubernetesProvider.CreateMizuApiServerPod(ctx, opts) if err != nil { - mizu.Log.Infof("Error creating mizu %s pod: %v", mizu.ApiServerPodName, err) return err } mizu.Log.Debugf("Successfully created API server pod: %s", mizu.ApiServerPodName) - apiServerService, err = kubernetesProvider.CreateService(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName, mizu.ApiServerPodName) + state.apiServerService, err = kubernetesProvider.CreateService(ctx, mizu.Config.ResourcesNamespace(), mizu.ApiServerPodName, mizu.ApiServerPodName) if err != nil { - mizu.Log.Infof("Error creating mizu %s service: %v", mizu.ApiServerPodName, err) return err } mizu.Log.Debugf("Successfully created service: %s", mizu.ApiServerPodName) @@ -183,7 +195,6 @@ func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Pro } func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) { - var compiledRegexSlice []*shared.SerializableRegexp if mizu.Config.Tap.PlainTextFilterRegexes != nil && len(mizu.Config.Tap.PlainTextFilterRegexes) > 0 { @@ -191,7 +202,6 @@ func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) { for _, regexStr := range mizu.Config.Tap.PlainTextFilterRegexes { compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr) if err != nil { - mizu.Log.Infof("Regex %s is invalid: %v", regexStr, err) return nil, err } compiledRegexSlice = append(compiledRegexSlice, compiledRegex) @@ -204,7 +214,7 @@ func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) { func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string) error { if len(nodeToTappedPodIPMap) > 0 { var serviceAccountName string - if mizuServiceAccountExists { + if state.mizuServiceAccountExists { serviceAccountName = mizu.ServiceAccountName } else { serviceAccountName = "" @@ -212,22 +222,20 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi if err := kubernetesProvider.ApplyMizuTapperDaemonSet( ctx, - mizu.ResourcesNamespace, + mizu.Config.ResourcesNamespace(), mizu.TapperDaemonSetName, mizu.Config.MizuImage, mizu.TapperPodName, - fmt.Sprintf("%s.%s.svc.cluster.local", apiServerService.Name, apiServerService.Namespace), + fmt.Sprintf("%s.%s.svc.cluster.local", state.apiServerService.Name, state.apiServerService.Namespace), nodeToTappedPodIPMap, serviceAccountName, mizu.Config.Tap.TapOutgoing(), ); err != nil { - mizu.Log.Infof("Error creating mizu tapper daemonset: %v", err) return err } mizu.Log.Debugf("Successfully created %v tappers", len(nodeToTappedPodIPMap)) } else { - if err := kubernetesProvider.RemoveDaemonSet(ctx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil { - mizu.Log.Errorf("Error deleting mizu tapper daemonset: %v", err) + if err := kubernetesProvider.RemoveDaemonSet(ctx, mizu.Config.ResourcesNamespace(), mizu.TapperDaemonSetName); err != nil { return err } } @@ -241,31 +249,73 @@ func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) { removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) defer cancel() - if err := kubernetesProvider.RemoveNamespace(removalCtx, mizu.ResourcesNamespace); err != nil { - mizu.Log.Infof("Error removing Namespace %s: %s (%v,%+v)", mizu.ResourcesNamespace, err, err, err) - return + if mizu.Config.IsOwnNamespace() { + if err := kubernetesProvider.RemoveNamespace(removalCtx, mizu.Config.ResourcesNamespace()); err != nil { + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Namespace %s: %v", mizu.Config.ResourcesNamespace(), errormessage.FormatError(err))) + return + } + } else { + if err := kubernetesProvider.RemovePod(removalCtx, mizu.Config.ResourcesNamespace(), mizu.ApiServerPodName); err != nil { + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Pod %s in namespace %s: %v", mizu.ApiServerPodName, mizu.Config.ResourcesNamespace(), errormessage.FormatError(err))) + } + + if err := kubernetesProvider.RemoveService(removalCtx, mizu.Config.ResourcesNamespace(), mizu.ApiServerPodName); err != nil { + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service %s in namespace %s: %v", mizu.ApiServerPodName, mizu.Config.ResourcesNamespace(), errormessage.FormatError(err))) + } + + if err := kubernetesProvider.RemoveDaemonSet(removalCtx, mizu.Config.ResourcesNamespace(), mizu.TapperDaemonSetName); err != nil { + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing DaemonSet %s in namespace %s: %v", mizu.TapperDaemonSetName, mizu.Config.ResourcesNamespace(), errormessage.FormatError(err))) + } + + if !state.doNotRemoveConfigMap { + if err := kubernetesProvider.RemoveConfigMap(removalCtx, mizu.Config.ResourcesNamespace(), mizu.ConfigMapName); err != nil { + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing ConfigMap %s in namespace %s: %v", mizu.ConfigMapName, mizu.Config.ResourcesNamespace(), errormessage.FormatError(err))) + } + } + } - if mizuServiceAccountExists { - if err := kubernetesProvider.RemoveNonNamespacedResources(removalCtx, mizu.ClusterRoleName, mizu.ClusterRoleBindingName); err != nil { - mizu.Log.Infof("Error removing non-namespaced resources: %s (%v,%+v)", err, err, err) - return + if state.mizuServiceAccountExists { + if mizu.Config.IsOwnNamespace() { + if err := kubernetesProvider.RemoveNonNamespacedResources(removalCtx, mizu.ClusterRoleName, mizu.ClusterRoleBindingName); err != nil { + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing non-namespaced resources: %v", errormessage.FormatError(err))) + return + } + } else { + if err := kubernetesProvider.RemoveServicAccount(removalCtx, mizu.Config.ResourcesNamespace(), mizu.ServiceAccountName); err != nil { + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service Account %s in namespace %s: %v", mizu.ServiceAccountName, mizu.Config.ResourcesNamespace(), errormessage.FormatError(err))) + return + } + + if err := kubernetesProvider.RemoveRole(removalCtx, mizu.Config.ResourcesNamespace(), mizu.RoleName); err != nil { + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Role %s in namespace %s: %v", mizu.RoleName, mizu.Config.ResourcesNamespace(), errormessage.FormatError(err))) + } + + if err := kubernetesProvider.RemoveRoleBinding(removalCtx, mizu.Config.ResourcesNamespace(), mizu.RoleBindingName); err != nil { + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing RoleBinding %s in namespace %s: %v", mizu.RoleBindingName, mizu.Config.ResourcesNamespace(), errormessage.FormatError(err))) + } } } + if mizu.Config.IsOwnNamespace() { + waitUntilNamespaceDeleted(removalCtx, cancel, kubernetesProvider) + } +} + +func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) { // Call cancel if a terminating signal was received. Allows user to skip the wait. go func() { - waitForFinish(removalCtx, cancel) + waitForFinish(ctx, cancel) }() - if err := kubernetesProvider.WaitUtilNamespaceDeleted(removalCtx, mizu.ResourcesNamespace); err != nil { + if err := kubernetesProvider.WaitUtilNamespaceDeleted(ctx, mizu.Config.ResourcesNamespace()); err != nil { switch { - case removalCtx.Err() == context.Canceled: + case ctx.Err() == context.Canceled: // Do nothing. User interrupted the wait. case err == wait.ErrWaitTimeout: - mizu.Log.Infof("Timeout while removing Namespace %s", mizu.ResourcesNamespace) + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Timeout while removing Namespace %s", mizu.Config.ResourcesNamespace())) default: - mizu.Log.Infof("Error while waiting for Namespace %s to be deleted: %s (%v,%+v)", mizu.ResourcesNamespace, err, err, err) + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error while waiting for Namespace %s to be deleted: %v", mizu.Config.ResourcesNamespace(), errormessage.FormatError(err))) } } } @@ -275,7 +325,7 @@ func reportTappedPods() { tappedPodsUrl := fmt.Sprintf("http://%s/status/tappedPods", mizuProxiedUrl) podInfos := make([]shared.PodInfo, 0) - for _, pod := range currentlyTappedPods { + for _, pod := range state.currentlyTappedPods { podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace}) } tapStatus := shared.TapStatus{Pods: podInfos} @@ -300,7 +350,7 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro restartTappers := func() { err, changeFound := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespace) if err != nil { - mizu.Log.Errorf("Error getting pods by regex: %s (%v,%+v)", err, err, err) + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error getting pods by regex: %v", errormessage.FormatError(err))) cancel() } @@ -311,13 +361,13 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro reportTappedPods() - nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods) + nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods) if err != nil { - mizu.Log.Errorf("Error building node to ips map: %s (%v,%+v)", err, err, err) + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error building node to ips map: %v", errormessage.FormatError(err))) cancel() } if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil { - mizu.Log.Errorf("Error updating daemonset: %s (%v,%+v)", err, err, err) + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating daemonset: %v", errormessage.FormatError(err))) cancel() } } @@ -356,10 +406,9 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro func updateCurrentlyTappedPods(kubernetesProvider *kubernetes.Provider, ctx context.Context, targetNamespace string) (error, bool) { changeFound := false if matchingPods, err := kubernetesProvider.GetAllRunningPodsMatchingRegex(ctx, mizu.Config.Tap.PodRegex(), targetNamespace); err != nil { - mizu.Log.Infof("Error getting pods by regex: %s (%v,%+v)", err, err, err) return err, false } else { - addedPods, removedPods := getPodArrayDiff(currentlyTappedPods, matchingPods) + addedPods, removedPods := getPodArrayDiff(state.currentlyTappedPods, matchingPods) for _, addedPod := range addedPods { changeFound = true mizu.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", addedPod.Name)) @@ -368,7 +417,7 @@ func updateCurrentlyTappedPods(kubernetesProvider *kubernetes.Provider, ctx cont changeFound = true mizu.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s", removedPod.Name)) } - currentlyTappedPods = matchingPods + state.currentlyTappedPods = matchingPods } return nil, changeFound @@ -401,7 +450,7 @@ func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod { func createProxyToApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) { podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.ApiServerPodName)) - added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, mizu.ResourcesNamespace), podExactRegex) + added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, mizu.Config.ResourcesNamespace()), podExactRegex) isPodReady := false timeAfter := time.After(25 * time.Second) for { @@ -420,24 +469,26 @@ func createProxyToApiServerPod(ctx context.Context, kubernetesProvider *kubernet if modifiedPod.Status.Phase == core.PodRunning && !isPodReady { isPodReady = true go func() { - err := kubernetes.StartProxy(kubernetesProvider, mizu.Config.Tap.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName) + err := kubernetes.StartProxy(kubernetesProvider, mizu.Config.Tap.GuiPort, mizu.Config.ResourcesNamespace(), mizu.ApiServerPodName) if err != nil { - mizu.Log.Errorf("Error occurred while running k8s proxy %v", err) + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v", errormessage.FormatError(err))) cancel() } }() - mizu.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort)) + mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort) + mizu.Log.Infof("Mizu is available at http://%s\n", mizuProxiedUrl) + time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready requestForAnalysis() reportTappedPods() } case <-timeAfter: if !isPodReady { - mizu.Log.Errorf("Error: %s pod was not ready in time", mizu.ApiServerPodName) + mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("%s pod was not ready in time", mizu.ApiServerPodName)) cancel() } case <-errorChan: - mizu.Log.Debugf("[ERROR] Agent creation, watching %v namespace", mizu.ResourcesNamespace) + mizu.Log.Debugf("[ERROR] Agent creation, watching %v namespace", mizu.Config.ResourcesNamespace()) cancel() } } @@ -465,23 +516,28 @@ func requestForAnalysis() { } } -func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool { - mizuRBACExists, err := kubernetesProvider.DoesServiceAccountExist(ctx, mizu.ResourcesNamespace, mizu.ServiceAccountName) +func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) (bool, error) { + mizuRBACExists, err := kubernetesProvider.DoesServiceAccountExist(ctx, mizu.Config.ResourcesNamespace(), mizu.ServiceAccountName) if err != nil { - mizu.Log.Infof("warning: could not ensure mizu rbac resources exist %v", err) - return false + return false, err } if !mizuRBACExists { - err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.ResourcesNamespace, mizu.ServiceAccountName, mizu.ClusterRoleName, mizu.ClusterRoleBindingName, mizu.RBACVersion) - if err != nil && !errors.IsAlreadyExists(err) { - mizu.Log.Infof("warning: could not create mizu rbac resources %v", err) - return false + if mizu.Config.IsOwnNamespace() { + err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.Config.ResourcesNamespace(), mizu.ServiceAccountName, mizu.ClusterRoleName, mizu.ClusterRoleBindingName, mizu.RBACVersion) + if err != nil { + return false, err + } + } else { + err := kubernetesProvider.CreateMizuRBACNamespaceRestricted(ctx, mizu.Config.ResourcesNamespace(), mizu.ServiceAccountName, mizu.RoleName, mizu.RoleBindingName, mizu.RBACVersion) + if err != nil { + return false, err + } } } - return true + return true, nil } -func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) (map[string][]string, error) { +func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) map[string][]string { nodeToTappedPodIPMap := make(map[string][]string, 0) for _, pod := range tappedPods { existingList := nodeToTappedPodIPMap[pod.Spec.NodeName] @@ -491,7 +547,7 @@ func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) (map[string][]string, e nodeToTappedPodIPMap[pod.Spec.NodeName] = append(nodeToTappedPodIPMap[pod.Spec.NodeName], pod.Status.PodIP) } } - return nodeToTappedPodIPMap, nil + return nodeToTappedPodIPMap } func waitForFinish(ctx context.Context, cancel context.CancelFunc) { diff --git a/cli/cmd/viewRunner.go b/cli/cmd/viewRunner.go index c15c71be7..cdc2119f5 100644 --- a/cli/cmd/viewRunner.go +++ b/cli/cmd/viewRunner.go @@ -27,7 +27,7 @@ func runMizuView() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - exists, err := kubernetesProvider.DoesServicesExist(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName) + exists, err := kubernetesProvider.DoesServicesExist(ctx, mizu.Config.ResourcesNamespace(), mizu.ApiServerPodName) if err != nil { panic(err) } @@ -45,7 +45,7 @@ func runMizuView() { mizu.Log.Infof("Found service %s, creating k8s proxy", mizu.ApiServerPodName) mizu.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.View.GuiPort)) - err = kubernetes.StartProxy(kubernetesProvider, mizu.Config.View.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName) + err = kubernetes.StartProxy(kubernetesProvider, mizu.Config.View.GuiPort, mizu.Config.ResourcesNamespace(), mizu.ApiServerPodName) if err != nil { mizu.Log.Infof("Error occured while running k8s proxy %v", err) } diff --git a/cli/errormessage/errormessage.go b/cli/errormessage/errormessage.go new file mode 100644 index 000000000..510581e8d --- /dev/null +++ b/cli/errormessage/errormessage.go @@ -0,0 +1,29 @@ +package errormessage + +import ( + "errors" + "fmt" + regexpsyntax "regexp/syntax" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" +) + +// formatError wraps error with a detailed message that is meant for the user. +// While the errors are meant to be displayed, they are not meant to be exported as classes outsite of CLI. +func FormatError(err error) error { + var errorNew error + if k8serrors.IsForbidden(err) { + errorNew = fmt.Errorf("Insufficient permissions: %w. Supply the required permission or control Mizu's access to namespaces by setting MizuNamespace in the config file or setting the tapped namespace with --set mizu-namespace=.", err) + } else if syntaxError, isSyntaxError := asRegexSyntaxError(err); isSyntaxError { + errorNew = fmt.Errorf("Regex %s is invalid: %w", syntaxError.Expr, err) + } else { + errorNew = err + } + + return errorNew +} + +func asRegexSyntaxError(err error) (*regexpsyntax.Error, bool) { + var syntaxError *regexpsyntax.Error + return syntaxError, errors.As(err, &syntaxError) +} diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index 38f6dc848..211d9ea1a 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -12,16 +12,13 @@ import ( "strconv" "github.com/up9inc/mizu/cli/mizu" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/homedir" - "github.com/up9inc/mizu/shared" core "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" resource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/watch" applyconfapp "k8s.io/client-go/applyconfigurations/apps/v1" @@ -33,9 +30,11 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" _ "k8s.io/client-go/plugin/pkg/client/auth/openstack" restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" _ "k8s.io/client-go/tools/portforward" watchtools "k8s.io/client-go/tools/watch" + "k8s.io/client-go/util/homedir" ) type Provider struct { @@ -126,8 +125,18 @@ func (provider *Provider) CreateNamespace(ctx context.Context, name string) (*co return provider.clientSet.CoreV1().Namespaces().Create(ctx, namespaceSpec, metav1.CreateOptions{}) } -func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, namespace string, podName string, podImage string, serviceAccountName string, mizuApiFilteringOptions *shared.TrafficFilteringOptions, maxEntriesDBSizeBytes int64) (*core.Pod, error) { - marshaledFilteringOptions, err := json.Marshal(mizuApiFilteringOptions) +type ApiServerOptions struct { + Namespace string + PodName string + PodImage string + ServiceAccountName string + IsNamespaceRestricted bool + MizuApiFilteringOptions *shared.TrafficFilteringOptions + MaxEntriesDBSizeBytes int64 +} + +func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiServerOptions) (*core.Pod, error) { + marshaledFilteringOptions, err := json.Marshal(opts.MizuApiFilteringOptions) if err != nil { return nil, err } @@ -138,32 +147,37 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, namespace cpuLimit, err := resource.ParseQuantity("750m") if err != nil { - return nil, errors.New(fmt.Sprintf("invalid cpu limit for %s container", podName)) + return nil, errors.New(fmt.Sprintf("invalid cpu limit for %s container", opts.PodName)) } memLimit, err := resource.ParseQuantity("512Mi") if err != nil { - return nil, errors.New(fmt.Sprintf("invalid memory limit for %s container", podName)) + return nil, errors.New(fmt.Sprintf("invalid memory limit for %s container", opts.PodName)) } cpuRequests, err := resource.ParseQuantity("50m") if err != nil { - return nil, errors.New(fmt.Sprintf("invalid cpu request for %s container", podName)) + return nil, errors.New(fmt.Sprintf("invalid cpu request for %s container", opts.PodName)) } memRequests, err := resource.ParseQuantity("50Mi") if err != nil { - return nil, errors.New(fmt.Sprintf("invalid memory request for %s container", podName)) + return nil, errors.New(fmt.Sprintf("invalid memory request for %s container", opts.PodName)) + } + + command := []string{"./mizuagent", "--api-server"} + if opts.IsNamespaceRestricted { + command = append(command, "--namespace", opts.Namespace) } pod := &core.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Namespace: namespace, - Labels: map[string]string{"app": podName}, + Name: opts.PodName, + Namespace: opts.Namespace, + Labels: map[string]string{"app": opts.PodName}, }, Spec: core.PodSpec{ Containers: []core.Container{ { - Name: podName, - Image: podImage, + Name: opts.PodName, + Image: opts.PodImage, ImagePullPolicy: core.PullAlways, VolumeMounts: []core.VolumeMount{ { @@ -171,7 +185,7 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, namespace MountPath: shared.RulePolicyPath, }, }, - Command: []string{"./mizuagent", "--api-server"}, + Command: command, Env: []core.EnvVar{ { Name: shared.HostModeEnvVar, @@ -183,7 +197,7 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, namespace }, { Name: shared.MaxEntriesDBSizeBytesEnvVar, - Value: strconv.FormatInt(maxEntriesDBSizeBytes, 10), + Value: strconv.FormatInt(opts.MaxEntriesDBSizeBytes, 10), }, }, Resources: core.ResourceRequirements{ @@ -211,10 +225,10 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, namespace }, } //define the service account only when it exists to prevent pod crash - if serviceAccountName != "" { - pod.Spec.ServiceAccountName = serviceAccountName + if opts.ServiceAccountName != "" { + pod.Spec.ServiceAccountName = opts.ServiceAccountName } - return provider.clientSet.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + return provider.clientSet.CoreV1().Pods(opts.Namespace).Create(ctx, pod, metav1.CreateOptions{}) } func (provider *Provider) CreateService(ctx context.Context, namespace string, serviceName string, appLabelValue string) (*core.Service, error) { @@ -234,7 +248,55 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s func (provider *Provider) DoesServiceAccountExist(ctx context.Context, namespace string, serviceAccountName string) (bool, error) { serviceAccount, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccountName, metav1.GetOptions{}) + return provider.doesResourceExist(serviceAccount, err) +} +func (provider *Provider) DoesConfigMapExist(ctx context.Context, namespace string, name string) (bool, error) { + resource, err := provider.clientSet.CoreV1().ConfigMaps(namespace).Get(ctx, name, metav1.GetOptions{}) + return provider.doesResourceExist(resource, err) +} + +func (provider *Provider) DoesServicesExist(ctx context.Context, namespace string, name string) (bool, error) { + resource, err := provider.clientSet.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{}) + return provider.doesResourceExist(resource, err) +} + +func (provider *Provider) DoesNamespaceExist(ctx context.Context, name string) (bool, error) { + resource, err := provider.clientSet.CoreV1().Namespaces().Get(ctx, name, metav1.GetOptions{}) + return provider.doesResourceExist(resource, err) +} + +func (provider *Provider) DoesClusterRoleExist(ctx context.Context, name string) (bool, error) { + resource, err := provider.clientSet.RbacV1().ClusterRoles().Get(ctx, name, metav1.GetOptions{}) + return provider.doesResourceExist(resource, err) +} + +func (provider *Provider) DoesClusterRoleBindingExist(ctx context.Context, name string) (bool, error) { + resource, err := provider.clientSet.RbacV1().ClusterRoleBindings().Get(ctx, name, metav1.GetOptions{}) + return provider.doesResourceExist(resource, err) +} + +func (provider *Provider) DoesRoleExist(ctx context.Context, namespace string, name string) (bool, error) { + resource, err := provider.clientSet.RbacV1().Roles(namespace).Get(ctx, name, metav1.GetOptions{}) + return provider.doesResourceExist(resource, err) +} + +func (provider *Provider) DoesRoleBindingExist(ctx context.Context, namespace string, name string) (bool, error) { + resource, err := provider.clientSet.RbacV1().RoleBindings(namespace).Get(ctx, name, metav1.GetOptions{}) + return provider.doesResourceExist(resource, err) +} + +func (provider *Provider) DoesPodExist(ctx context.Context, namespace string, name string) (bool, error) { + resource, err := provider.clientSet.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) + return provider.doesResourceExist(resource, err) +} + +func (provider *Provider) DoesDaemonSetExist(ctx context.Context, namespace string, name string) (bool, error) { + resource, err := provider.clientSet.AppsV1().DaemonSets(namespace).Get(ctx, name, metav1.GetOptions{}) + return provider.doesResourceExist(resource, err) +} + +func (provider *Provider) doesResourceExist(resource interface{}, err error) (bool, error) { var statusError *k8serrors.StatusError if errors.As(err, &statusError) { // expected behavior when resource does not exist @@ -245,22 +307,7 @@ func (provider *Provider) DoesServiceAccountExist(ctx context.Context, namespace if err != nil { return false, err } - return serviceAccount != nil, nil -} - -func (provider *Provider) DoesServicesExist(ctx context.Context, namespace string, serviceName string) (bool, error) { - service, err := provider.clientSet.CoreV1().Services(namespace).Get(ctx, serviceName, metav1.GetOptions{}) - - var statusError *k8serrors.StatusError - if errors.As(err, &statusError) { - if statusError.ErrStatus.Reason == metav1.StatusReasonNotFound { - return false, nil - } - } - if err != nil { - return false, err - } - return service != nil, nil + return resource != nil, nil } func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, serviceAccountName string, clusterRoleName string, clusterRoleBindingName string, version string) error { @@ -317,8 +364,62 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, return nil } +func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context, namespace string, serviceAccountName string, roleName string, roleBindingName string, version string) error { + serviceAccount := &core.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceAccountName, + Namespace: namespace, + Labels: map[string]string{"mizu-cli-version": version}, + }, + } + role := &rbac.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleName, + Labels: map[string]string{"mizu-cli-version": version}, + }, + Rules: []rbac.PolicyRule{ + { + APIGroups: []string{"", "extensions", "apps"}, + Resources: []string{"pods", "services", "endpoints"}, + Verbs: []string{"list", "get", "watch"}, + }, + }, + } + roleBinding := &rbac.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleBindingName, + Labels: map[string]string{"mizu-cli-version": version}, + }, + RoleRef: rbac.RoleRef{ + Name: roleName, + Kind: "Role", + APIGroup: "rbac.authorization.k8s.io", + }, + Subjects: []rbac.Subject{ + { + Kind: "ServiceAccount", + Name: serviceAccountName, + Namespace: namespace, + }, + }, + } + _, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Create(ctx, serviceAccount, metav1.CreateOptions{}) + if err != nil { + return err + } + _, err = provider.clientSet.RbacV1().Roles(namespace).Create(ctx, role, metav1.CreateOptions{}) + if err != nil { + return err + } + _, err = provider.clientSet.RbacV1().RoleBindings(namespace).Create(ctx, roleBinding, metav1.CreateOptions{}) + if err != nil { + return err + } + return nil +} + func (provider *Provider) RemoveNamespace(ctx context.Context, name string) error { - if isFound, err := provider.CheckNamespaceExists(ctx, name); err != nil { + if isFound, err := provider.DoesNamespaceExist(ctx, name); err != nil { return err } else if !isFound { return nil @@ -340,7 +441,7 @@ func (provider *Provider) RemoveNonNamespacedResources(ctx context.Context, clus } func (provider *Provider) RemoveClusterRole(ctx context.Context, name string) error { - if isFound, err := provider.CheckClusterRoleExists(ctx, name); err != nil { + if isFound, err := provider.DoesClusterRoleExist(ctx, name); err != nil { return err } else if !isFound { return nil @@ -350,7 +451,7 @@ func (provider *Provider) RemoveClusterRole(ctx context.Context, name string) er } func (provider *Provider) RemoveClusterRoleBinding(ctx context.Context, name string) error { - if isFound, err := provider.CheckClusterRoleBindingExists(ctx, name); err != nil { + if isFound, err := provider.DoesClusterRoleBindingExist(ctx, name); err != nil { return err } else if !isFound { return nil @@ -359,8 +460,38 @@ func (provider *Provider) RemoveClusterRoleBinding(ctx context.Context, name str return provider.clientSet.RbacV1().ClusterRoleBindings().Delete(ctx, name, metav1.DeleteOptions{}) } +func (provider *Provider) RemoveRoleBinding(ctx context.Context, namespace string, name string) error { + if isFound, err := provider.DoesRoleBindingExist(ctx, namespace, name); err != nil { + return err + } else if !isFound { + return nil + } + + return provider.clientSet.RbacV1().RoleBindings(namespace).Delete(ctx, name, metav1.DeleteOptions{}) +} + +func (provider *Provider) RemoveRole(ctx context.Context, namespace string, name string) error { + if isFound, err := provider.DoesRoleExist(ctx, namespace, name); err != nil { + return err + } else if !isFound { + return nil + } + + return provider.clientSet.RbacV1().Roles(namespace).Delete(ctx, name, metav1.DeleteOptions{}) +} + +func (provider *Provider) RemoveServicAccount(ctx context.Context, namespace string, name string) error { + if isFound, err := provider.DoesServiceAccountExist(ctx, namespace, name); err != nil { + return err + } else if !isFound { + return nil + } + + return provider.clientSet.CoreV1().ServiceAccounts(namespace).Delete(ctx, name, metav1.DeleteOptions{}) +} + func (provider *Provider) RemovePod(ctx context.Context, namespace string, podName string) error { - if isFound, err := provider.CheckPodExists(ctx, namespace, podName); err != nil { + if isFound, err := provider.DoesPodExist(ctx, namespace, podName); err != nil { return err } else if !isFound { return nil @@ -369,8 +500,18 @@ func (provider *Provider) RemovePod(ctx context.Context, namespace string, podNa return provider.clientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{}) } +func (provider *Provider) RemoveConfigMap(ctx context.Context, namespace string, configMapName string) error { + if isFound, err := provider.DoesConfigMapExist(ctx, namespace, configMapName); err != nil { + return err + } else if !isFound { + return nil + } + + return provider.clientSet.CoreV1().ConfigMaps(namespace).Delete(ctx, configMapName, metav1.DeleteOptions{}) +} + func (provider *Provider) RemoveService(ctx context.Context, namespace string, serviceName string) error { - if isFound, err := provider.CheckServiceExists(ctx, namespace, serviceName); err != nil { + if isFound, err := provider.DoesServicesExist(ctx, namespace, serviceName); err != nil { return err } else if !isFound { return nil @@ -380,7 +521,7 @@ func (provider *Provider) RemoveService(ctx context.Context, namespace string, s } func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, daemonSetName string) error { - if isFound, err := provider.CheckDaemonSetExists(ctx, namespace, daemonSetName); err != nil { + if isFound, err := provider.DoesDaemonSetExist(ctx, namespace, daemonSetName); err != nil { return err } else if !isFound { return nil @@ -389,138 +530,11 @@ func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, return provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{}) } -func (provider *Provider) RemoveConfigMap(ctx context.Context, namespace string, configMapName string) error { - if isFound, err := provider.CheckConfigMapExists(ctx, namespace, configMapName); err != nil { - return err - } else if !isFound { - return nil - } - return provider.clientSet.CoreV1().ConfigMaps(namespace).Delete(ctx, configMapName, metav1.DeleteOptions{}) -} - -func (provider *Provider) CheckNamespaceExists(ctx context.Context, name string) (bool, error) { - listOptions := metav1.ListOptions{ - FieldSelector: fmt.Sprintf("metadata.name=%s", name), - Limit: 1, - } - resourceList, err := provider.clientSet.CoreV1().Namespaces().List(ctx, listOptions) - if err != nil { - return false, err - } - - if len(resourceList.Items) > 0 { - return true, nil - } - - return false, nil -} - -func (provider *Provider) CheckClusterRoleExists(ctx context.Context, name string) (bool, error) { - listOptions := metav1.ListOptions{ - FieldSelector: fmt.Sprintf("metadata.name=%s", name), - Limit: 1, - } - resourceList, err := provider.clientSet.RbacV1().ClusterRoles().List(ctx, listOptions) - if err != nil { - return false, err - } - - if len(resourceList.Items) > 0 { - return true, nil - } - - return false, nil -} - -func (provider *Provider) CheckClusterRoleBindingExists(ctx context.Context, name string) (bool, error) { - listOptions := metav1.ListOptions{ - FieldSelector: fmt.Sprintf("metadata.name=%s", name), - Limit: 1, - } - resourceList, err := provider.clientSet.RbacV1().ClusterRoleBindings().List(ctx, listOptions) - if err != nil { - return false, err - } - - if len(resourceList.Items) > 0 { - return true, nil - } - - return false, nil -} - -func (provider *Provider) CheckPodExists(ctx context.Context, namespace string, name string) (bool, error) { - listOptions := metav1.ListOptions{ - FieldSelector: fmt.Sprintf("metadata.name=%s", name), - Limit: 1, - } - resourceList, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, listOptions) - if err != nil { - return false, err - } - - if len(resourceList.Items) > 0 { - return true, nil - } - - return false, nil -} - -func (provider *Provider) CheckServiceExists(ctx context.Context, namespace string, name string) (bool, error) { - listOptions := metav1.ListOptions{ - FieldSelector: fmt.Sprintf("metadata.name=%s", name), - Limit: 1, - } - resourceList, err := provider.clientSet.CoreV1().Services(namespace).List(ctx, listOptions) - if err != nil { - return false, err - } - - if len(resourceList.Items) > 0 { - return true, nil - } - - return false, nil -} - -func (provider *Provider) CheckDaemonSetExists(ctx context.Context, namespace string, name string) (bool, error) { - listOptions := metav1.ListOptions{ - FieldSelector: fmt.Sprintf("metadata.name=%s", name), - Limit: 1, - } - resourceList, err := provider.clientSet.AppsV1().DaemonSets(namespace).List(ctx, listOptions) - if err != nil { - return false, err - } - - if len(resourceList.Items) > 0 { - return true, nil - } - - return false, nil -} - -func (provider *Provider) CheckConfigMapExists(ctx context.Context, namespace string, name string) (bool, error) { - listOptions := metav1.ListOptions{ - FieldSelector: fmt.Sprintf("metadata.name=%s", name), - Limit: 1, - } - resourceList, err := provider.clientSet.CoreV1().ConfigMaps(namespace).List(ctx, listOptions) - if err != nil { - return false, err - } - - if len(resourceList.Items) > 0 { - return true, nil - } - - return false, nil -} - -func (provider *Provider) ApplyConfigMap(ctx context.Context, namespace string, configMapName string, data string) error { +func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, data string) error { if data == "" { return nil } + configMapData := make(map[string]string, 0) configMapData[shared.RulePolicyFileName] = data configMap := &core.ConfigMap{ @@ -534,14 +548,11 @@ func (provider *Provider) ApplyConfigMap(ctx context.Context, namespace string, }, Data: configMapData, } - _, err := provider.clientSet.CoreV1().ConfigMaps(namespace).Create(ctx, configMap, metav1.CreateOptions{}) - var statusError *k8serrors.StatusError - if errors.As(err, &statusError) { - if statusError.ErrStatus.Reason == metav1.StatusReasonForbidden { - return fmt.Errorf("User not authorized to create configmap, --test-rules will be ignored") - } + if _, err := provider.clientSet.CoreV1().ConfigMaps(namespace).Create(ctx, configMap, metav1.CreateOptions{}); err != nil { + return err } - return err + + return nil } func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool) error { @@ -671,7 +682,7 @@ func (provider *Provider) GetAllRunningPodsMatchingRegex(ctx context.Context, re matchingPods = append(matchingPods, pod) } } - return matchingPods, err + return matchingPods, nil } func getClientSet(config *restclient.Config) *kubernetes.Clientset { diff --git a/cli/mizu/configStruct.go b/cli/mizu/configStruct.go index d3c1d5864..a3e358e1b 100644 --- a/cli/mizu/configStruct.go +++ b/cli/mizu/configStruct.go @@ -7,14 +7,31 @@ import ( ) type ConfigStruct struct { - Tap configStructs.TapConfig `yaml:"tap"` - Fetch configStructs.FetchConfig `yaml:"fetch"` - Version configStructs.VersionConfig `yaml:"version"` - View configStructs.ViewConfig `yaml:"view"` - MizuImage string `yaml:"mizu-image"` - Telemetry bool `yaml:"telemetry" default:"true"` + Tap configStructs.TapConfig `yaml:"tap"` + Fetch configStructs.FetchConfig `yaml:"fetch"` + Version configStructs.VersionConfig `yaml:"version"` + View configStructs.ViewConfig `yaml:"view"` + MizuImage string `yaml:"mizu-image"` + MizuNamespace string `yaml:"mizu-namespace"` + Telemetry bool `yaml:"telemetry" default:"true"` } func (config *ConfigStruct) SetDefaults() { config.MizuImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", Branch, SemVer) } + +func (config *ConfigStruct) ResourcesNamespace() string { + if config.MizuNamespace == "" { + return ResourcesDefaultNamespace + } + + return config.MizuNamespace +} + +func (config *ConfigStruct) IsOwnNamespace() bool { + if config.MizuNamespace == "" { + return true + } + + return false +} diff --git a/cli/mizu/consts.go b/cli/mizu/consts.go index f9ade7e5a..e1489a05d 100644 --- a/cli/mizu/consts.go +++ b/cli/mizu/consts.go @@ -14,15 +14,17 @@ var ( ) const ( - ApiServerPodName = "mizu-api-server" - ClusterRoleBindingName = "mizu-cluster-role-binding" - ClusterRoleName = "mizu-cluster-role" - K8sAllNamespaces = "" - ResourcesNamespace = "mizu" - ServiceAccountName = "mizu-service-account" - TapperDaemonSetName = "mizu-tapper-daemon-set" - TapperPodName = "mizu-tapper" - ConfigMapName = "mizu-policy" + ApiServerPodName = "mizu-api-server" + ClusterRoleBindingName = "mizu-cluster-role-binding" + ClusterRoleName = "mizu-cluster-role" + K8sAllNamespaces = "" + ResourcesDefaultNamespace = "mizu" + RoleBindingName = "mizu-role-binding" + RoleName = "mizu-role" + ServiceAccountName = "mizu-service-account" + TapperDaemonSetName = "mizu-tapper-daemon-set" + TapperPodName = "mizu-tapper" + ConfigMapName = "mizu-policy" ) func getMizuFolderPath() string { diff --git a/cli/uiUtils/colors.go b/cli/uiUtils/colors.go index e4c684dda..c7b0887c4 100644 --- a/cli/uiUtils/colors.go +++ b/cli/uiUtils/colors.go @@ -2,12 +2,14 @@ package uiUtils const ( - Black = "\033[1;30m%s\033[0m" - Red = "\033[1;31m%s\033[0m" - Green = "\033[1;32m%s\033[0m" - Yellow = "\033[1;33m%s\033[0m" - Purple = "\033[1;34m%s\033[0m" - Magenta = "\033[1;35m%s\033[0m" - Teal = "\033[1;36m%s\033[0m" - White = "\033[1;37m%s\033[0m" -) \ No newline at end of file + Black = "\033[1;30m%s\033[0m" + Red = "\033[1;31m%s\033[0m" + Green = "\033[1;32m%s\033[0m" + Yellow = "\033[1;33m%s\033[0m" + Purple = "\033[1;34m%s\033[0m" + Magenta = "\033[1;35m%s\033[0m" + Teal = "\033[1;36m%s\033[0m" + White = "\033[1;37m%s\033[0m" + Error = Red + Warning = Yellow +) diff --git a/examples/roles/permissions-all-namespaces-without-ip-resolution.yaml b/examples/roles/permissions-all-namespaces-without-ip-resolution.yaml index 3252abb89..c4e809ac3 100644 --- a/examples/roles/permissions-all-namespaces-without-ip-resolution.yaml +++ b/examples/roles/permissions-all-namespaces-without-ip-resolution.yaml @@ -7,16 +7,16 @@ metadata: rules: - apiGroups: [""] resources: ["pods"] - verbs: ["list", "watch", "create"] + verbs: ["list", "watch", "create", "delete"] - apiGroups: [""] resources: ["services"] - verbs: ["create"] + verbs: ["create", "delete"] - apiGroups: ["apps"] resources: ["daemonsets"] - verbs: ["create", "patch"] + verbs: ["create", "patch", "delete"] - apiGroups: [""] resources: ["namespaces"] - verbs: ["list", "watch", "create", "delete"] + verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: [""] resources: ["services/proxy"] verbs: ["get"] diff --git a/examples/roles/permissions-all-namespaces.yaml b/examples/roles/permissions-all-namespaces.yaml index 0f3ba3dff..ff1060df0 100644 --- a/examples/roles/permissions-all-namespaces.yaml +++ b/examples/roles/permissions-all-namespaces.yaml @@ -6,28 +6,34 @@ metadata: rules: - apiGroups: [""] resources: ["pods"] - verbs: ["get", "list", "watch", "create"] + verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: [""] resources: ["services"] - verbs: ["get", "list", "watch", "create"] + verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: ["apps"] resources: ["daemonsets"] - verbs: ["create", "patch"] + verbs: ["create", "patch", "delete"] - apiGroups: [""] resources: ["namespaces"] - verbs: ["list", "watch", "create", "delete"] + verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: [""] resources: ["services/proxy"] verbs: ["get"] - apiGroups: [""] resources: ["serviceaccounts"] - verbs: ["get", "create"] + verbs: ["get", "create", "delete"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["clusterroles"] - verbs: ["list", "create", "delete"] + verbs: ["get", "create", "delete"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["clusterrolebindings"] - verbs: ["list", "create", "delete"] + verbs: ["get", "create", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles"] + verbs: ["get", "create", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["rolebindings"] + verbs: ["get", "create", "delete"] - apiGroups: ["apps", "extensions"] resources: ["pods"] verbs: ["get", "list", "watch"] diff --git a/examples/roles/permissions-ns-with-validation.yaml b/examples/roles/permissions-ns-with-validation.yaml new file mode 100644 index 000000000..e2d6863ec --- /dev/null +++ b/examples/roles/permissions-ns-with-validation.yaml @@ -0,0 +1,54 @@ +# This example shows the roles required for a user to be able to use Mizu in a single namespace. +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mizu-runner-role + namespace: user1 +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "delete"] +- apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch", "create", "delete"] +- apiGroups: ["apps"] + resources: ["daemonsets"] + verbs: ["get", "create", "patch", "delete"] +- apiGroups: [""] + resources: ["services/proxy"] + verbs: ["get"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "delete"] +- apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get", "create", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles"] + verbs: ["get", "create", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["rolebindings"] + verbs: ["get", "create", "delete"] +- apiGroups: ["apps", "extensions"] + resources: ["pods"] + verbs: ["get", "list", "watch"] +- apiGroups: ["apps", "extensions"] + resources: ["services"] + verbs: ["get", "list", "watch"] +- apiGroups: ["", "apps", "extensions"] + resources: ["endpoints"] + verbs: ["get", "list", "watch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mizu-runner-rolebindings + namespace: user1 +subjects: +- kind: User + name: user1 + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: Role + name: mizu-runner-role + apiGroup: rbac.authorization.k8s.io diff --git a/examples/roles/permissions-ns-without-ip-resolution.yaml b/examples/roles/permissions-ns-without-ip-resolution.yaml new file mode 100644 index 000000000..4293f8979 --- /dev/null +++ b/examples/roles/permissions-ns-without-ip-resolution.yaml @@ -0,0 +1,33 @@ +# This example shows the roles required for a user to be able to use Mizu in a single namespace with IP resolution disabled. +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mizu-runner-role + namespace: user1 +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "delete"] +- apiGroups: [""] + resources: ["services"] + verbs: ["get", "create", "delete"] +- apiGroups: ["apps"] + resources: ["daemonsets"] + verbs: ["get", "create", "patch", "delete"] +- apiGroups: [""] + resources: ["services/proxy"] + verbs: ["get"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mizu-runner-rolebindings + namespace: user1 +subjects: +- kind: User + name: user1 + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: Role + name: mizu-runner-role + apiGroup: rbac.authorization.k8s.io diff --git a/examples/roles/permissions-ns.yaml b/examples/roles/permissions-ns.yaml new file mode 100644 index 000000000..da60a02e1 --- /dev/null +++ b/examples/roles/permissions-ns.yaml @@ -0,0 +1,51 @@ +# This example shows the roles required for a user to be able to use Mizu in a single namespace. +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mizu-runner-role + namespace: user1 +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "delete"] +- apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch", "create", "delete"] +- apiGroups: ["apps"] + resources: ["daemonsets"] + verbs: ["get", "create", "patch", "delete"] +- apiGroups: [""] + resources: ["services/proxy"] + verbs: ["get"] +- apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get", "create", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles"] + verbs: ["get", "create", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["rolebindings"] + verbs: ["get", "create", "delete"] +- apiGroups: ["apps", "extensions"] + resources: ["pods"] + verbs: ["get", "list", "watch"] +- apiGroups: ["apps", "extensions"] + resources: ["services"] + verbs: ["get", "list", "watch"] +- apiGroups: ["", "apps", "extensions"] + resources: ["endpoints"] + verbs: ["get", "list", "watch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mizu-runner-rolebindings + namespace: user1 +subjects: +- kind: User + name: user1 + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: Role + name: mizu-runner-role + apiGroup: rbac.authorization.k8s.io