✨ Add Ingress (#1357)

* ✨ Add `Ingress`

* ⚡ Rewrite the target in `Ingress`

* ⚡ Fix the path of front pod in `Ingress`

* ✨ Add `IngressConfig` struct

* ⚡ Generate the correct Helm chart based on `tap.ingress` field of `values.yaml`

cmd/helmChart.go

@@ -177,6 +177,12 @@ var workerDaemonSetMappings = map[string]interface{}{
 	"spec.template.spec.containers[0].command[4]":                "{{ .Values.tap.proxy.worker.srvport }}",
 	"spec.template.spec.containers[0].command[6]":                "{{ .Values.tap.packetcapture }}",
 }
+var ingressClassMappings = serviceAccountMappings
+var ingressMappings = map[string]interface{}{
+	"metadata.namespace": "{{ .Values.tap.selfnamespace }}",
+	"spec.rules[0].host": "{{ .Values.tap.ingress.host }}",
+	"spec.tls":           "{{ .Values.tap.ingress.tls | toYaml }}",
+}
 
 func init() {
 	rootCmd.AddCommand(helmChartCmd)
@@ -193,6 +199,8 @@ func runHelmChart() {
 		frontService,
 		persistentVolume,
 		workerDaemonSet,
+		ingressClass,
+		ingress,
 		err := generateManifests()
 	if err != nil {
 		log.Error().Err(err).Send()
@@ -210,6 +218,8 @@ func runHelmChart() {
 		"07-front-service.yaml":           template(frontService, frontServiceMappings),
 		"08-persistent-volume-claim.yaml": template(persistentVolume, persistentVolumeMappings),
 		"09-worker-daemon-set.yaml":       template(workerDaemonSet, workerDaemonSetMappings),
+		"10-ingress-class.yaml":           template(ingressClass, ingressClassMappings),
+		"11-ingress.yaml":                 template(ingress, ingressMappings),
 	})
 	if err != nil {
 		log.Error().Err(err).Send()
@@ -319,6 +329,16 @@ func handleDaemonSetManifest(manifest string) string {
 	return strings.Join(lines, "\n")
 }
 
+func handleIngressClass(manifest string) string {
+	return fmt.Sprintf("{{- if .Values.tap.ingress.enabled }}\n%s{{- end }}\n", manifest)
+}
+
+func handleIngress(manifest string) string {
+	manifest = strings.Replace(manifest, "'{{ .Values.tap.ingress.tls | toYaml }}'", "{{ .Values.tap.ingress.tls | toYaml }}", 1)
+
+	return handleIngressClass(manifest)
+}
+
 func dumpHelmChart(objects map[string]interface{}) error {
 	folder := filepath.Join(".", "helm-chart")
 	templatesFolder := filepath.Join(folder, "templates")
@@ -363,6 +383,14 @@ func dumpHelmChart(objects map[string]interface{}) error {
 			manifest = handleDaemonSetManifest(manifest)
 		}
 
+		if filename == "10-ingress-class.yaml" {
+			manifest = handleIngressClass(manifest)
+		}
+
+		if filename == "11-ingress.yaml" {
+			manifest = handleIngress(manifest)
+		}
+
 		path := filepath.Join(templatesFolder, filename)
 		err = os.WriteFile(path, []byte(manifestHeader+manifest), 0644)
 		if err != nil {

cmd/manifests.go

@@ -15,6 +15,7 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	v1 "k8s.io/api/core/v1"
+	networking "k8s.io/api/networking/v1"
 	rbac "k8s.io/api/rbac/v1"
 )
 
@@ -52,6 +53,8 @@ func runManifests() {
 		frontService,
 		persistentVolume,
 		workerDaemonSet,
+		ingressClass,
+		ingress,
 		err := generateManifests()
 	if err != nil {
 		log.Error().Err(err).Send()
@@ -70,6 +73,8 @@ func runManifests() {
 			"07-front-service.yaml":           frontService,
 			"08-persistent-volume-claim.yaml": persistentVolume,
 			"09-worker-daemon-set.yaml":       workerDaemonSet,
+			"10-ingress-class.yaml":           ingressClass,
+			"11-ingress.yaml":                 ingress,
 		})
 	} else {
 		err = printManifests([]interface{}{
@@ -101,6 +106,8 @@ func generateManifests() (
 	frontService *v1.Service,
 	persistentVolumeClaim *v1.PersistentVolumeClaim,
 	workerDaemonSet *kubernetes.DaemonSet,
+	ingressClass *networking.IngressClass,
+	ingress *networking.Ingress,
 	err error,
 ) {
 	config.Config.License = ""
@@ -173,6 +180,9 @@ func generateManifests() (
 		return
 	}
 
+	ingressClass = kubernetesProvider.BuildIngressClass()
+	ingress = kubernetesProvider.BuildIngress()
+
 	config.Config.Tap.PersistentStorage = persistentStorage
 
 	return

cmd/tap.go

@@ -60,5 +60,6 @@ func init() {
 	tapCmd.Flags().Bool(configStructs.ServiceMeshLabel, defaultTapConfig.ServiceMesh, "Capture the encrypted traffic if the cluster is configured with a service mesh and with mTLS")
 	tapCmd.Flags().Bool(configStructs.TlsLabel, defaultTapConfig.Tls, "Capture the traffic that's encrypted with OpenSSL or Go crypto/tls libraries")
 	tapCmd.Flags().Bool(configStructs.IgnoreTaintedLabel, defaultTapConfig.IgnoreTainted, "Ignore tainted pods while running Worker DaemonSet")
+	tapCmd.Flags().Bool(configStructs.IngressEnabledLabel, defaultTapConfig.Ingress.Enabled, "Enable Ingress")
 	tapCmd.Flags().Bool(configStructs.DebugLabel, defaultTapConfig.Debug, "Enable the debug mode")
 }

config/configStructs/tapConfig.go

@@ -5,6 +5,7 @@ import (
 	"regexp"
 
 	v1 "k8s.io/api/core/v1"
+	networking "k8s.io/api/networking/v1"
 )
 
 const (
@@ -25,6 +26,7 @@ const (
 	ServiceMeshLabel       = "servicemesh"
 	TlsLabel               = "tls"
 	IgnoreTaintedLabel     = "ignoreTainted"
+	IngressEnabledLabel    = "ingress-enabled"
 	DebugLabel             = "debug"
 	ContainerPort          = 80
 	ContainerPortStr       = "80"
@@ -78,6 +80,12 @@ type ResourcesConfig struct {
 	Hub    ResourceRequirements `yaml:"hub"`
 }
 
+type IngressConfig struct {
+	Enabled bool                    `yaml:"enabled" default:"false"`
+	Host    string                  `yaml:"host" default:"ks.svc.cluster.local"`
+	TLS     []networking.IngressTLS `yaml:"tls"`
+}
+
 type TapConfig struct {
 	Docker            DockerConfig          `yaml:"docker"`
 	Proxy             ProxyConfig           `yaml:"proxy"`
@@ -96,6 +104,7 @@ type TapConfig struct {
 	IgnoreTainted     bool                  `yaml:"ignoreTainted" default:"false"`
 	ResourceLabels    map[string]string     `yaml:"resourceLabels" default:"{}"`
 	NodeSelectorTerms []v1.NodeSelectorTerm `yaml:"nodeSelectorTerms" default:"[]"`
+	Ingress           IngressConfig         `yaml:"ingress"`
 	Debug             bool                  `yaml:"debug" default:"false"`
 }
 

helm-chart/templates/02-cluster-role.yaml

@@ -20,6 +20,7 @@ rules:
       - services
       - endpoints
       - persistentvolumeclaims
+      - ingresses
     verbs:
       - list
       - get

helm-chart/templates/05-hub-service.yaml

@@ -16,6 +16,6 @@ spec:
       targetPort: 80
   selector:
     app: kubeshark-hub
-  type: ClusterIP
+  type: NodePort
 status:
   loadBalancer: {}

helm-chart/templates/07-front-service.yaml

@@ -16,6 +16,6 @@ spec:
       targetPort: 80
   selector:
     app: kubeshark-front
-  type: ClusterIP
+  type: NodePort
 status:
   loadBalancer: {}

helm-chart/templates/10-ingress-class.yaml

@@ -0,0 +1,16 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY KUBESHARK CLI. DO NOT EDIT!
+---
+{{- if .Values.tap.ingress.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  creationTimestamp: null
+  labels:
+    kubeshark-cli-version: v1
+    kubeshark-created-by: kubeshark
+    kubeshark-managed-by: kubeshark
+  name: kubeshark-ingress-class
+  namespace: '{{ .Values.tap.selfnamespace }}'
+spec:
+  controller: k8s.io/ingress-nginx
+{{- end }}

helm-chart/templates/11-ingress.yaml

@@ -0,0 +1,39 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY KUBESHARK CLI. DO NOT EDIT!
+---
+{{- if .Values.tap.ingress.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/rewrite-target: /$2
+  creationTimestamp: null
+  labels:
+    kubeshark-cli-version: v1
+    kubeshark-created-by: kubeshark
+    kubeshark-managed-by: kubeshark
+  name: kubeshark-ingress
+  namespace: '{{ .Values.tap.selfnamespace }}'
+spec:
+  ingressClassName: kubeshark-ingress-class
+  rules:
+    - host: '{{ .Values.tap.ingress.host }}'
+      http:
+        paths:
+          - backend:
+              service:
+                name: kubeshark-hub
+                port:
+                  number: 80
+            path: /api(/|$)(.*)
+            pathType: Prefix
+          - backend:
+              service:
+                name: kubeshark-front
+                port:
+                  number: 80
+            path: /()(.*)
+            pathType: Prefix
+  tls: {{ .Values.tap.ingress.tls | toYaml }}
+status:
+  loadBalancer: {}
+{{- end }}

helm-chart/values.yaml

@@ -43,6 +43,10 @@ tap:
   ignoreTainted: false
   resourceLabels: {}
   nodeSelectorTerms: []
+  ingress:
+    enabled: false
+    host: ks.svc.cluster.local
+    tls: []
   debug: false
 logs:
   file: ""

kubernetes/consts.go

@@ -16,6 +16,8 @@ const (
 	WorkerPodName              = SelfResourcesPrefix + "worker"
 	PersistentVolumeName       = SelfResourcesPrefix + "persistent-volume"
 	PersistentVolumeClaimName  = SelfResourcesPrefix + "persistent-volume-claim"
+	IngressName                = SelfResourcesPrefix + "ingress"
+	IngressClassName           = SelfResourcesPrefix + "ingress-class"
 	PersistentVolumeHostPath   = "/app/data"
 	MinKubernetesServerVersion = "1.16.0"
 )

kubernetes/provider.go

@@ -20,6 +20,7 @@ import (
 	"github.com/rs/zerolog/log"
 	auth "k8s.io/api/authorization/v1"
 	core "k8s.io/api/core/v1"
+	networking "k8s.io/api/networking/v1"
 	rbac "k8s.io/api/rbac/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -324,6 +325,10 @@ func (provider *Provider) BuildFrontPod(opts *PodOptions, hubHost string, hubPor
 	volumeMounts := []core.VolumeMount{}
 	volumes := []core.Volume{}
 
+	if config.Config.Tap.Ingress.Enabled {
+		hubPort = "80/api"
+	}
+
 	containers := []core.Container{
 		{
 			Name:  opts.PodName,
@@ -431,7 +436,7 @@ func (provider *Provider) BuildHubService(namespace string) *core.Service {
 					Port:       configStructs.ContainerPort,
 				},
 			},
-			Type:     core.ServiceTypeClusterIP,
+			Type:     core.ServiceTypeNodePort,
 			Selector: map[string]string{"app": HubServiceName},
 		},
 	}
@@ -456,12 +461,20 @@ func (provider *Provider) BuildFrontService(namespace string) *core.Service {
 					Port:       configStructs.ContainerPort,
 				},
 			},
-			Type:     core.ServiceTypeClusterIP,
+			Type:     core.ServiceTypeNodePort,
 			Selector: map[string]string{"app": FrontServiceName},
 		},
 	}
 }
 
+func (provider *Provider) CreateIngressClass(ctx context.Context, ingressClass *networking.IngressClass) (*networking.IngressClass, error) {
+	return provider.clientSet.NetworkingV1().IngressClasses().Create(ctx, ingressClass, metav1.CreateOptions{})
+}
+
+func (provider *Provider) CreateIngress(ctx context.Context, namespace string, ingress *networking.Ingress) (*networking.Ingress, error) {
+	return provider.clientSet.NetworkingV1().Ingresses(namespace).Create(ctx, ingress, metav1.CreateOptions{})
+}
+
 func (provider *Provider) CreateService(ctx context.Context, namespace string, service *core.Service) (*core.Service, error) {
 	return provider.clientSet.CoreV1().Services(namespace).Create(ctx, service, metav1.CreateOptions{})
 }
@@ -534,6 +547,86 @@ func (provider *Provider) doesResourceExist(resource interface{}, err error) (bo
 	return resource != nil, nil
 }
 
+func (provider *Provider) BuildIngressClass() *networking.IngressClass {
+	return &networking.IngressClass{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "IngressClass",
+			APIVersion: "networking.k8s.io/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      IngressClassName,
+			Namespace: config.Config.Tap.SelfNamespace,
+			Labels: buildWithDefaultLabels(map[string]string{
+				fmt.Sprintf("%s-cli-version", misc.Program): misc.RBACVersion,
+			}, provider),
+		},
+		Spec: networking.IngressClassSpec{
+			Controller: "k8s.io/ingress-nginx",
+		},
+	}
+}
+
+func (provider *Provider) BuildIngress() *networking.Ingress {
+	pathTypePrefix := networking.PathTypePrefix
+	ingressClassName := IngressClassName
+
+	return &networking.Ingress{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Ingress",
+			APIVersion: "networking.k8s.io/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      IngressName,
+			Namespace: config.Config.Tap.SelfNamespace,
+			Labels: buildWithDefaultLabels(map[string]string{
+				fmt.Sprintf("%s-cli-version", misc.Program): misc.RBACVersion,
+			}, provider),
+			Annotations: map[string]string{
+				"nginx.ingress.kubernetes.io/rewrite-target": "/$2",
+			},
+		},
+		Spec: networking.IngressSpec{
+			IngressClassName: &ingressClassName,
+			TLS:              config.Config.Tap.Ingress.TLS,
+			Rules: []networking.IngressRule{
+				{
+					Host: config.Config.Tap.Ingress.Host,
+					IngressRuleValue: networking.IngressRuleValue{
+						HTTP: &networking.HTTPIngressRuleValue{
+							Paths: []networking.HTTPIngressPath{
+								{
+									Path:     "/api(/|$)(.*)",
+									PathType: &pathTypePrefix,
+									Backend: networking.IngressBackend{
+										Service: &networking.IngressServiceBackend{
+											Name: HubServiceName,
+											Port: networking.ServiceBackendPort{
+												Number: configStructs.ContainerPort,
+											},
+										},
+									},
+								},
+								{
+									Path:     "/()(.*)",
+									PathType: &pathTypePrefix,
+									Backend: networking.IngressBackend{
+										Service: &networking.IngressServiceBackend{
+											Name: FrontServiceName,
+											Port: networking.ServiceBackendPort{
+												Number: configStructs.ContainerPort,
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
 func (provider *Provider) BuildServiceAccount() *core.ServiceAccount {
 	return &core.ServiceAccount{
 		TypeMeta: metav1.TypeMeta{
@@ -575,6 +668,7 @@ func (provider *Provider) BuildClusterRole() *rbac.ClusterRole {
 					"services",
 					"endpoints",
 					"persistentvolumeclaims",
+					"ingresses",
 				},
 				Verbs: []string{
 					"list",
@@ -634,6 +728,11 @@ func (provider *Provider) CreateSelfRBAC(ctx context.Context, namespace string)
 	return nil
 }
 
+func (provider *Provider) RemoveIngressClass(ctx context.Context, name string) error {
+	err := provider.clientSet.NetworkingV1().IngressClasses().Delete(ctx, name, metav1.DeleteOptions{})
+	return provider.handleRemovalError(err)
+}
+
 func (provider *Provider) RemoveNamespace(ctx context.Context, name string) error {
 	err := provider.clientSet.CoreV1().Namespaces().Delete(ctx, name, metav1.DeleteOptions{})
 	return provider.handleRemovalError(err)

manifests/02-cluster-role.yaml

@@ -20,6 +20,7 @@ rules:
       - services
       - endpoints
       - persistentvolumeclaims
+      - ingresses
     verbs:
       - list
       - get

manifests/05-hub-service.yaml

@@ -16,6 +16,6 @@ spec:
       targetPort: 80
   selector:
     app: kubeshark-hub
-  type: ClusterIP
+  type: NodePort
 status:
   loadBalancer: {}

manifests/07-front-service.yaml

@@ -16,6 +16,6 @@ spec:
       targetPort: 80
   selector:
     app: kubeshark-front
-  type: ClusterIP
+  type: NodePort
 status:
   loadBalancer: {}

manifests/10-ingress-class.yaml

@@ -0,0 +1,14 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY KUBESHARK CLI. DO NOT EDIT!
+---
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  creationTimestamp: null
+  labels:
+    kubeshark-cli-version: v1
+    kubeshark-created-by: kubeshark
+    kubeshark-managed-by: kubeshark
+  name: kubeshark-ingress-class
+  namespace: kubeshark
+spec:
+  controller: k8s.io/ingress-nginx

manifests/11-ingress.yaml

@@ -0,0 +1,36 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY KUBESHARK CLI. DO NOT EDIT!
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/rewrite-target: /$2
+  creationTimestamp: null
+  labels:
+    kubeshark-cli-version: v1
+    kubeshark-created-by: kubeshark
+    kubeshark-managed-by: kubeshark
+  name: kubeshark-ingress
+  namespace: kubeshark
+spec:
+  ingressClassName: kubeshark-ingress-class
+  rules:
+    - host: ks.svc.cluster.local
+      http:
+        paths:
+          - backend:
+              service:
+                name: kubeshark-hub
+                port:
+                  number: 80
+            path: /api(/|$)(.*)
+            pathType: Prefix
+          - backend:
+              service:
+                name: kubeshark-front
+                port:
+                  number: 80
+            path: /()(.*)
+            pathType: Prefix
+status:
+  loadBalancer: {}

resources/cleanResources.go

@@ -35,6 +35,11 @@ func CleanUpSelfResources(ctx context.Context, cancel context.CancelFunc, kubern
 func cleanUpNonRestrictedMode(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, selfResourcesNamespace string) []string {
 	leftoverResources := make([]string, 0)
 
+	if err := kubernetesProvider.RemoveIngressClass(ctx, kubernetes.IngressClassName); err != nil {
+		resourceDesc := kubernetes.IngressClassName
+		handleDeletionError(err, resourceDesc, &leftoverResources)
+	}
+
 	if err := kubernetesProvider.RemoveNamespace(ctx, selfResourcesNamespace); err != nil {
 		resourceDesc := fmt.Sprintf("Namespace %s", selfResourcesNamespace)
 		handleDeletionError(err, resourceDesc, &leftoverResources)

resources/createResources.go

@@ -58,21 +58,30 @@ func CreateHubResources(ctx context.Context, kubernetesProvider *kubernetes.Prov
 		return selfServiceAccountExists, err
 	}
 
-	// TODO: Why the port values need to be 80?
 	_, err = kubernetesProvider.CreateService(ctx, selfNamespace, kubernetesProvider.BuildHubService(selfNamespace))
 	if err != nil {
 		return selfServiceAccountExists, err
 	}
-
 	log.Info().Str("service", kubernetes.HubServiceName).Msg("Successfully created a service.")
 
 	_, err = kubernetesProvider.CreateService(ctx, selfNamespace, kubernetesProvider.BuildFrontService(selfNamespace))
 	if err != nil {
 		return selfServiceAccountExists, err
 	}
-
 	log.Info().Str("service", kubernetes.FrontServiceName).Msg("Successfully created a service.")
 
+	_, err = kubernetesProvider.CreateIngressClass(ctx, kubernetesProvider.BuildIngressClass())
+	if err != nil {
+		return selfServiceAccountExists, err
+	}
+	log.Info().Str("ingress-class", kubernetes.IngressClassName).Msg("Successfully created an ingress class.")
+
+	_, err = kubernetesProvider.CreateIngress(ctx, selfNamespace, kubernetesProvider.BuildIngress())
+	if err != nil {
+		return selfServiceAccountExists, err
+	}
+	log.Info().Str("ingress", kubernetes.IngressName).Msg("Successfully created an ingress.")
+
 	return selfServiceAccountExists, nil
 }