From ad9dfbce40317d4e89a1afe52f4dee8ec2638158 Mon Sep 17 00:00:00 2001 From: "M. Mert Yildiran" Date: Tue, 16 May 2023 09:46:47 -0700 Subject: [PATCH] :sparkles: Add `Ingress` (#1357) * :sparkles: Add `Ingress` * :zap: Rewrite the target in `Ingress` * :zap: Fix the path of front pod in `Ingress` * :sparkles: Add `IngressConfig` struct * :zap: Generate the correct Helm chart based on `tap.ingress` field of `values.yaml` --- cmd/helmChart.go | 28 ++++++ cmd/manifests.go | 10 ++ cmd/tap.go | 1 + config/configStructs/tapConfig.go | 9 ++ helm-chart/templates/02-cluster-role.yaml | 1 + helm-chart/templates/05-hub-service.yaml | 2 +- helm-chart/templates/07-front-service.yaml | 2 +- helm-chart/templates/10-ingress-class.yaml | 16 ++++ helm-chart/templates/11-ingress.yaml | 39 ++++++++ helm-chart/values.yaml | 4 + kubernetes/consts.go | 2 + kubernetes/provider.go | 103 ++++++++++++++++++++- manifests/02-cluster-role.yaml | 1 + manifests/05-hub-service.yaml | 2 +- manifests/07-front-service.yaml | 2 +- manifests/10-ingress-class.yaml | 14 +++ manifests/11-ingress.yaml | 36 +++++++ resources/cleanResources.go | 5 + resources/createResources.go | 15 ++- 19 files changed, 283 insertions(+), 9 deletions(-) create mode 100644 helm-chart/templates/10-ingress-class.yaml create mode 100644 helm-chart/templates/11-ingress.yaml create mode 100644 manifests/10-ingress-class.yaml create mode 100644 manifests/11-ingress.yaml diff --git a/cmd/helmChart.go b/cmd/helmChart.go index d5e8dd4f2..60232f3bf 100644 --- a/cmd/helmChart.go +++ b/cmd/helmChart.go @@ -177,6 +177,12 @@ var workerDaemonSetMappings = map[string]interface{}{ "spec.template.spec.containers[0].command[4]": "{{ .Values.tap.proxy.worker.srvport }}", "spec.template.spec.containers[0].command[6]": "{{ .Values.tap.packetcapture }}", } +var ingressClassMappings = serviceAccountMappings +var ingressMappings = map[string]interface{}{ + "metadata.namespace": "{{ .Values.tap.selfnamespace }}", + "spec.rules[0].host": "{{ .Values.tap.ingress.host }}", + "spec.tls": "{{ .Values.tap.ingress.tls | toYaml }}", +} func init() { rootCmd.AddCommand(helmChartCmd) @@ -193,6 +199,8 @@ func runHelmChart() { frontService, persistentVolume, workerDaemonSet, + ingressClass, + ingress, err := generateManifests() if err != nil { log.Error().Err(err).Send() @@ -210,6 +218,8 @@ func runHelmChart() { "07-front-service.yaml": template(frontService, frontServiceMappings), "08-persistent-volume-claim.yaml": template(persistentVolume, persistentVolumeMappings), "09-worker-daemon-set.yaml": template(workerDaemonSet, workerDaemonSetMappings), + "10-ingress-class.yaml": template(ingressClass, ingressClassMappings), + "11-ingress.yaml": template(ingress, ingressMappings), }) if err != nil { log.Error().Err(err).Send() @@ -319,6 +329,16 @@ func handleDaemonSetManifest(manifest string) string { return strings.Join(lines, "\n") } +func handleIngressClass(manifest string) string { + return fmt.Sprintf("{{- if .Values.tap.ingress.enabled }}\n%s{{- end }}\n", manifest) +} + +func handleIngress(manifest string) string { + manifest = strings.Replace(manifest, "'{{ .Values.tap.ingress.tls | toYaml }}'", "{{ .Values.tap.ingress.tls | toYaml }}", 1) + + return handleIngressClass(manifest) +} + func dumpHelmChart(objects map[string]interface{}) error { folder := filepath.Join(".", "helm-chart") templatesFolder := filepath.Join(folder, "templates") @@ -363,6 +383,14 @@ func dumpHelmChart(objects map[string]interface{}) error { manifest = handleDaemonSetManifest(manifest) } + if filename == "10-ingress-class.yaml" { + manifest = handleIngressClass(manifest) + } + + if filename == "11-ingress.yaml" { + manifest = handleIngress(manifest) + } + path := filepath.Join(templatesFolder, filename) err = os.WriteFile(path, []byte(manifestHeader+manifest), 0644) if err != nil { diff --git a/cmd/manifests.go b/cmd/manifests.go index 6f539f0d3..2f1d81ecd 100644 --- a/cmd/manifests.go +++ b/cmd/manifests.go @@ -15,6 +15,7 @@ import ( "github.com/rs/zerolog/log" "github.com/spf13/cobra" v1 "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1" rbac "k8s.io/api/rbac/v1" ) @@ -52,6 +53,8 @@ func runManifests() { frontService, persistentVolume, workerDaemonSet, + ingressClass, + ingress, err := generateManifests() if err != nil { log.Error().Err(err).Send() @@ -70,6 +73,8 @@ func runManifests() { "07-front-service.yaml": frontService, "08-persistent-volume-claim.yaml": persistentVolume, "09-worker-daemon-set.yaml": workerDaemonSet, + "10-ingress-class.yaml": ingressClass, + "11-ingress.yaml": ingress, }) } else { err = printManifests([]interface{}{ @@ -101,6 +106,8 @@ func generateManifests() ( frontService *v1.Service, persistentVolumeClaim *v1.PersistentVolumeClaim, workerDaemonSet *kubernetes.DaemonSet, + ingressClass *networking.IngressClass, + ingress *networking.Ingress, err error, ) { config.Config.License = "" @@ -173,6 +180,9 @@ func generateManifests() ( return } + ingressClass = kubernetesProvider.BuildIngressClass() + ingress = kubernetesProvider.BuildIngress() + config.Config.Tap.PersistentStorage = persistentStorage return diff --git a/cmd/tap.go b/cmd/tap.go index e4b4018dc..51d2844ac 100644 --- a/cmd/tap.go +++ b/cmd/tap.go @@ -60,5 +60,6 @@ func init() { tapCmd.Flags().Bool(configStructs.ServiceMeshLabel, defaultTapConfig.ServiceMesh, "Capture the encrypted traffic if the cluster is configured with a service mesh and with mTLS") tapCmd.Flags().Bool(configStructs.TlsLabel, defaultTapConfig.Tls, "Capture the traffic that's encrypted with OpenSSL or Go crypto/tls libraries") tapCmd.Flags().Bool(configStructs.IgnoreTaintedLabel, defaultTapConfig.IgnoreTainted, "Ignore tainted pods while running Worker DaemonSet") + tapCmd.Flags().Bool(configStructs.IngressEnabledLabel, defaultTapConfig.Ingress.Enabled, "Enable Ingress") tapCmd.Flags().Bool(configStructs.DebugLabel, defaultTapConfig.Debug, "Enable the debug mode") } diff --git a/config/configStructs/tapConfig.go b/config/configStructs/tapConfig.go index 18593b8e7..6a8bc4e42 100644 --- a/config/configStructs/tapConfig.go +++ b/config/configStructs/tapConfig.go @@ -5,6 +5,7 @@ import ( "regexp" v1 "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1" ) const ( @@ -25,6 +26,7 @@ const ( ServiceMeshLabel = "servicemesh" TlsLabel = "tls" IgnoreTaintedLabel = "ignoreTainted" + IngressEnabledLabel = "ingress-enabled" DebugLabel = "debug" ContainerPort = 80 ContainerPortStr = "80" @@ -78,6 +80,12 @@ type ResourcesConfig struct { Hub ResourceRequirements `yaml:"hub"` } +type IngressConfig struct { + Enabled bool `yaml:"enabled" default:"false"` + Host string `yaml:"host" default:"ks.svc.cluster.local"` + TLS []networking.IngressTLS `yaml:"tls"` +} + type TapConfig struct { Docker DockerConfig `yaml:"docker"` Proxy ProxyConfig `yaml:"proxy"` @@ -96,6 +104,7 @@ type TapConfig struct { IgnoreTainted bool `yaml:"ignoreTainted" default:"false"` ResourceLabels map[string]string `yaml:"resourceLabels" default:"{}"` NodeSelectorTerms []v1.NodeSelectorTerm `yaml:"nodeSelectorTerms" default:"[]"` + Ingress IngressConfig `yaml:"ingress"` Debug bool `yaml:"debug" default:"false"` } diff --git a/helm-chart/templates/02-cluster-role.yaml b/helm-chart/templates/02-cluster-role.yaml index f2e117904..1c8399d6b 100644 --- a/helm-chart/templates/02-cluster-role.yaml +++ b/helm-chart/templates/02-cluster-role.yaml @@ -20,6 +20,7 @@ rules: - services - endpoints - persistentvolumeclaims + - ingresses verbs: - list - get diff --git a/helm-chart/templates/05-hub-service.yaml b/helm-chart/templates/05-hub-service.yaml index 71879eae2..ca082482a 100644 --- a/helm-chart/templates/05-hub-service.yaml +++ b/helm-chart/templates/05-hub-service.yaml @@ -16,6 +16,6 @@ spec: targetPort: 80 selector: app: kubeshark-hub - type: ClusterIP + type: NodePort status: loadBalancer: {} diff --git a/helm-chart/templates/07-front-service.yaml b/helm-chart/templates/07-front-service.yaml index 056624fd2..149066628 100644 --- a/helm-chart/templates/07-front-service.yaml +++ b/helm-chart/templates/07-front-service.yaml @@ -16,6 +16,6 @@ spec: targetPort: 80 selector: app: kubeshark-front - type: ClusterIP + type: NodePort status: loadBalancer: {} diff --git a/helm-chart/templates/10-ingress-class.yaml b/helm-chart/templates/10-ingress-class.yaml new file mode 100644 index 000000000..5ce20bf5c --- /dev/null +++ b/helm-chart/templates/10-ingress-class.yaml @@ -0,0 +1,16 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY KUBESHARK CLI. DO NOT EDIT! +--- +{{- if .Values.tap.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + creationTimestamp: null + labels: + kubeshark-cli-version: v1 + kubeshark-created-by: kubeshark + kubeshark-managed-by: kubeshark + name: kubeshark-ingress-class + namespace: '{{ .Values.tap.selfnamespace }}' +spec: + controller: k8s.io/ingress-nginx +{{- end }} diff --git a/helm-chart/templates/11-ingress.yaml b/helm-chart/templates/11-ingress.yaml new file mode 100644 index 000000000..9d2bf6838 --- /dev/null +++ b/helm-chart/templates/11-ingress.yaml @@ -0,0 +1,39 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY KUBESHARK CLI. DO NOT EDIT! +--- +{{- if .Values.tap.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$2 + creationTimestamp: null + labels: + kubeshark-cli-version: v1 + kubeshark-created-by: kubeshark + kubeshark-managed-by: kubeshark + name: kubeshark-ingress + namespace: '{{ .Values.tap.selfnamespace }}' +spec: + ingressClassName: kubeshark-ingress-class + rules: + - host: '{{ .Values.tap.ingress.host }}' + http: + paths: + - backend: + service: + name: kubeshark-hub + port: + number: 80 + path: /api(/|$)(.*) + pathType: Prefix + - backend: + service: + name: kubeshark-front + port: + number: 80 + path: /()(.*) + pathType: Prefix + tls: {{ .Values.tap.ingress.tls | toYaml }} +status: + loadBalancer: {} +{{- end }} diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 8b0c0319c..e765f12a1 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -43,6 +43,10 @@ tap: ignoreTainted: false resourceLabels: {} nodeSelectorTerms: [] + ingress: + enabled: false + host: ks.svc.cluster.local + tls: [] debug: false logs: file: "" diff --git a/kubernetes/consts.go b/kubernetes/consts.go index 1f57e63b8..6cee83295 100644 --- a/kubernetes/consts.go +++ b/kubernetes/consts.go @@ -16,6 +16,8 @@ const ( WorkerPodName = SelfResourcesPrefix + "worker" PersistentVolumeName = SelfResourcesPrefix + "persistent-volume" PersistentVolumeClaimName = SelfResourcesPrefix + "persistent-volume-claim" + IngressName = SelfResourcesPrefix + "ingress" + IngressClassName = SelfResourcesPrefix + "ingress-class" PersistentVolumeHostPath = "/app/data" MinKubernetesServerVersion = "1.16.0" ) diff --git a/kubernetes/provider.go b/kubernetes/provider.go index 7f82f7dc2..e751990cb 100644 --- a/kubernetes/provider.go +++ b/kubernetes/provider.go @@ -20,6 +20,7 @@ import ( "github.com/rs/zerolog/log" auth "k8s.io/api/authorization/v1" core "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1" rbac "k8s.io/api/rbac/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -324,6 +325,10 @@ func (provider *Provider) BuildFrontPod(opts *PodOptions, hubHost string, hubPor volumeMounts := []core.VolumeMount{} volumes := []core.Volume{} + if config.Config.Tap.Ingress.Enabled { + hubPort = "80/api" + } + containers := []core.Container{ { Name: opts.PodName, @@ -431,7 +436,7 @@ func (provider *Provider) BuildHubService(namespace string) *core.Service { Port: configStructs.ContainerPort, }, }, - Type: core.ServiceTypeClusterIP, + Type: core.ServiceTypeNodePort, Selector: map[string]string{"app": HubServiceName}, }, } @@ -456,12 +461,20 @@ func (provider *Provider) BuildFrontService(namespace string) *core.Service { Port: configStructs.ContainerPort, }, }, - Type: core.ServiceTypeClusterIP, + Type: core.ServiceTypeNodePort, Selector: map[string]string{"app": FrontServiceName}, }, } } +func (provider *Provider) CreateIngressClass(ctx context.Context, ingressClass *networking.IngressClass) (*networking.IngressClass, error) { + return provider.clientSet.NetworkingV1().IngressClasses().Create(ctx, ingressClass, metav1.CreateOptions{}) +} + +func (provider *Provider) CreateIngress(ctx context.Context, namespace string, ingress *networking.Ingress) (*networking.Ingress, error) { + return provider.clientSet.NetworkingV1().Ingresses(namespace).Create(ctx, ingress, metav1.CreateOptions{}) +} + func (provider *Provider) CreateService(ctx context.Context, namespace string, service *core.Service) (*core.Service, error) { return provider.clientSet.CoreV1().Services(namespace).Create(ctx, service, metav1.CreateOptions{}) } @@ -534,6 +547,86 @@ func (provider *Provider) doesResourceExist(resource interface{}, err error) (bo return resource != nil, nil } +func (provider *Provider) BuildIngressClass() *networking.IngressClass { + return &networking.IngressClass{ + TypeMeta: metav1.TypeMeta{ + Kind: "IngressClass", + APIVersion: "networking.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: IngressClassName, + Namespace: config.Config.Tap.SelfNamespace, + Labels: buildWithDefaultLabels(map[string]string{ + fmt.Sprintf("%s-cli-version", misc.Program): misc.RBACVersion, + }, provider), + }, + Spec: networking.IngressClassSpec{ + Controller: "k8s.io/ingress-nginx", + }, + } +} + +func (provider *Provider) BuildIngress() *networking.Ingress { + pathTypePrefix := networking.PathTypePrefix + ingressClassName := IngressClassName + + return &networking.Ingress{ + TypeMeta: metav1.TypeMeta{ + Kind: "Ingress", + APIVersion: "networking.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: IngressName, + Namespace: config.Config.Tap.SelfNamespace, + Labels: buildWithDefaultLabels(map[string]string{ + fmt.Sprintf("%s-cli-version", misc.Program): misc.RBACVersion, + }, provider), + Annotations: map[string]string{ + "nginx.ingress.kubernetes.io/rewrite-target": "/$2", + }, + }, + Spec: networking.IngressSpec{ + IngressClassName: &ingressClassName, + TLS: config.Config.Tap.Ingress.TLS, + Rules: []networking.IngressRule{ + { + Host: config.Config.Tap.Ingress.Host, + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + { + Path: "/api(/|$)(.*)", + PathType: &pathTypePrefix, + Backend: networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: HubServiceName, + Port: networking.ServiceBackendPort{ + Number: configStructs.ContainerPort, + }, + }, + }, + }, + { + Path: "/()(.*)", + PathType: &pathTypePrefix, + Backend: networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: FrontServiceName, + Port: networking.ServiceBackendPort{ + Number: configStructs.ContainerPort, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + func (provider *Provider) BuildServiceAccount() *core.ServiceAccount { return &core.ServiceAccount{ TypeMeta: metav1.TypeMeta{ @@ -575,6 +668,7 @@ func (provider *Provider) BuildClusterRole() *rbac.ClusterRole { "services", "endpoints", "persistentvolumeclaims", + "ingresses", }, Verbs: []string{ "list", @@ -634,6 +728,11 @@ func (provider *Provider) CreateSelfRBAC(ctx context.Context, namespace string) return nil } +func (provider *Provider) RemoveIngressClass(ctx context.Context, name string) error { + err := provider.clientSet.NetworkingV1().IngressClasses().Delete(ctx, name, metav1.DeleteOptions{}) + return provider.handleRemovalError(err) +} + func (provider *Provider) RemoveNamespace(ctx context.Context, name string) error { err := provider.clientSet.CoreV1().Namespaces().Delete(ctx, name, metav1.DeleteOptions{}) return provider.handleRemovalError(err) diff --git a/manifests/02-cluster-role.yaml b/manifests/02-cluster-role.yaml index e2aa81771..5826702a7 100644 --- a/manifests/02-cluster-role.yaml +++ b/manifests/02-cluster-role.yaml @@ -20,6 +20,7 @@ rules: - services - endpoints - persistentvolumeclaims + - ingresses verbs: - list - get diff --git a/manifests/05-hub-service.yaml b/manifests/05-hub-service.yaml index 94863fa89..9d35f171f 100644 --- a/manifests/05-hub-service.yaml +++ b/manifests/05-hub-service.yaml @@ -16,6 +16,6 @@ spec: targetPort: 80 selector: app: kubeshark-hub - type: ClusterIP + type: NodePort status: loadBalancer: {} diff --git a/manifests/07-front-service.yaml b/manifests/07-front-service.yaml index 3fab9583c..945b0624e 100644 --- a/manifests/07-front-service.yaml +++ b/manifests/07-front-service.yaml @@ -16,6 +16,6 @@ spec: targetPort: 80 selector: app: kubeshark-front - type: ClusterIP + type: NodePort status: loadBalancer: {} diff --git a/manifests/10-ingress-class.yaml b/manifests/10-ingress-class.yaml new file mode 100644 index 000000000..e6e386a00 --- /dev/null +++ b/manifests/10-ingress-class.yaml @@ -0,0 +1,14 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY KUBESHARK CLI. DO NOT EDIT! +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + creationTimestamp: null + labels: + kubeshark-cli-version: v1 + kubeshark-created-by: kubeshark + kubeshark-managed-by: kubeshark + name: kubeshark-ingress-class + namespace: kubeshark +spec: + controller: k8s.io/ingress-nginx diff --git a/manifests/11-ingress.yaml b/manifests/11-ingress.yaml new file mode 100644 index 000000000..0e3ceb63e --- /dev/null +++ b/manifests/11-ingress.yaml @@ -0,0 +1,36 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY KUBESHARK CLI. DO NOT EDIT! +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$2 + creationTimestamp: null + labels: + kubeshark-cli-version: v1 + kubeshark-created-by: kubeshark + kubeshark-managed-by: kubeshark + name: kubeshark-ingress + namespace: kubeshark +spec: + ingressClassName: kubeshark-ingress-class + rules: + - host: ks.svc.cluster.local + http: + paths: + - backend: + service: + name: kubeshark-hub + port: + number: 80 + path: /api(/|$)(.*) + pathType: Prefix + - backend: + service: + name: kubeshark-front + port: + number: 80 + path: /()(.*) + pathType: Prefix +status: + loadBalancer: {} diff --git a/resources/cleanResources.go b/resources/cleanResources.go index 1c8283938..c29963847 100644 --- a/resources/cleanResources.go +++ b/resources/cleanResources.go @@ -35,6 +35,11 @@ func CleanUpSelfResources(ctx context.Context, cancel context.CancelFunc, kubern func cleanUpNonRestrictedMode(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, selfResourcesNamespace string) []string { leftoverResources := make([]string, 0) + if err := kubernetesProvider.RemoveIngressClass(ctx, kubernetes.IngressClassName); err != nil { + resourceDesc := kubernetes.IngressClassName + handleDeletionError(err, resourceDesc, &leftoverResources) + } + if err := kubernetesProvider.RemoveNamespace(ctx, selfResourcesNamespace); err != nil { resourceDesc := fmt.Sprintf("Namespace %s", selfResourcesNamespace) handleDeletionError(err, resourceDesc, &leftoverResources) diff --git a/resources/createResources.go b/resources/createResources.go index 83ef2e9ee..0fa607547 100644 --- a/resources/createResources.go +++ b/resources/createResources.go @@ -58,21 +58,30 @@ func CreateHubResources(ctx context.Context, kubernetesProvider *kubernetes.Prov return selfServiceAccountExists, err } - // TODO: Why the port values need to be 80? _, err = kubernetesProvider.CreateService(ctx, selfNamespace, kubernetesProvider.BuildHubService(selfNamespace)) if err != nil { return selfServiceAccountExists, err } - log.Info().Str("service", kubernetes.HubServiceName).Msg("Successfully created a service.") _, err = kubernetesProvider.CreateService(ctx, selfNamespace, kubernetesProvider.BuildFrontService(selfNamespace)) if err != nil { return selfServiceAccountExists, err } - log.Info().Str("service", kubernetes.FrontServiceName).Msg("Successfully created a service.") + _, err = kubernetesProvider.CreateIngressClass(ctx, kubernetesProvider.BuildIngressClass()) + if err != nil { + return selfServiceAccountExists, err + } + log.Info().Str("ingress-class", kubernetes.IngressClassName).Msg("Successfully created an ingress class.") + + _, err = kubernetesProvider.CreateIngress(ctx, selfNamespace, kubernetesProvider.BuildIngress()) + if err != nil { + return selfServiceAccountExists, err + } + log.Info().Str("ingress", kubernetes.IngressName).Msg("Successfully created an ingress.") + return selfServiceAccountExists, nil }