Load pf-ring kernel module in init container (#1476)

* Load kernel module in init container

* Update docs

* Update formatting

* Add pre-stop hook to unload pf_ring module

* Enable hook only on kernel module enabled

* fix template

* Use sidecontainer to unload pf_ring

* Add requirements for tracer into structs

* fix values

* fix typo

---------

Co-authored-by: Alon Girmonsky <1990761+alongir@users.noreply.github.com>

config/configStruct.go

@@ -16,14 +16,6 @@ const (
 func CreateDefaultConfig() ConfigStruct {
 	return ConfigStruct{
 		Tap: configStructs.TapConfig{
-			KernelModule: configStructs.KernelModuleConfig{
-				KernelMappings: []configStructs.KernelMapping{
-					{
-						ContainerImage: "kubeshark/pf-ring-module:${KERNEL_FULL_VERSION}",
-						Regexp:         "^.+$",
-					},
-				},
-			},
 			NodeSelectorTerms: []v1.NodeSelectorTerm{
 				{
 					MatchExpressions: []v1.NodeSelectorRequirement{

config/configStructs/tapConfig.go

@@ -79,6 +79,7 @@ type DockerConfig struct {
 type ResourcesConfig struct {
 	Worker ResourceRequirements `yaml:"worker" json:"worker"`
 	Hub    ResourceRequirements `yaml:"hub" json:"hub"`
+	Tracer ResourceRequirements `yaml:"tracer" json:"tracer"`
 }
 
 type AuthConfig struct {
@@ -113,16 +114,10 @@ type CapabilitiesConfig struct {
 	EBPFCapture        []string `yaml:"ebpfCapture" json:"ebpfCapture"  default:"[]"`
 }
 
-type KernelMapping struct {
-	Regexp         string `yaml:"regexp" json:"regexp"`
-	ContainerImage string `yaml:"containerImage" json:"containerImage"`
-}
-
 type KernelModuleConfig struct {
-	Enabled         bool            `yaml:"enabled" json:"enabled" default:"true"`
+	Enabled         bool   `yaml:"enabled" json:"enabled" default:"true"`
-	Mode            string          `yaml:"mode" json:"mode" default:"auto"`
+	Image           string `yaml:"image" json:"image" default:"kubeshark/pf-ring-module:all"`
-	KernelMappings  []KernelMapping `yaml:"kernelMappings" json:"kernelMappings"`
+	UnloadOnDestroy bool   `yaml:"unloadOnDestroy" json:"unloadOnDestroy" default:"false"`
-	ImageRepoSecret string          `yaml:"imageRepoSecret" json:"imageRepoSecret"`
 }
 
 type MetricsConfig struct {

helm-chart/PF_RING.md

@@ -2,16 +2,14 @@
 
 <!-- TOC -->
 
-- [PF_RING](#pf_ring)
+- [PF\_RING](#pf_ring)
-    - [Overview](#overview)
+  - [Overview](#overview)
-    - [Provisioning mode](#provisioning-mode)
+  - [Loading PF\_RING module on Kubernetes nodes](#loading-pf_ring-module-on-kubernetes-nodes)
-    - [Selection of Provisioning Mode](#selection-of-provisioning-mode)
+    - [Pre-built kernel module exists and external egress allowed](#pre-built-kernel-module-exists-and-external-egress-allowed)
-        - [Pre-built kernel module exists and external egress allowed](#pre-built-kernel-module-exists-and-external-egress-allowed)
+    - [Pre-built kernel module doesn't exist or external egress isn't allowed](#pre-built-kernel-module-doesnt-exist-or-external-egress-isnt-allowed)
-        - [Pre-built kernel module doesn't exist or external egress isn't allowed](#pre-built-kernel-module-doesnt-exist-or-external-egress-isnt-allowed)
+  - [Appendix A: PF\_RING kernel module compilation](#appendix-a-pf_ring-kernel-module-compilation)
-            - [Steps to Use kmm with Custom Containers](#steps-to-use-kmm-with-custom-containers)
+    - [Automated complilation](#automated-complilation)
-    - [Appendix A: PF_RING kernel module compilation](#appendix-a-pf_ring-kernel-module-compilation)
+    - [Manual compilation](#manual-compilation)
-        - [Automated complilation](#automated-complilation)
-        - [Manual compilation](#manual-compilation)
 
 <!-- /TOC -->
 
@@ -21,39 +19,26 @@ PF_RING™ is an advanced Linux kernel module and user-space framework designed
 
 For comprehensive information on PF_RING™, please visit the [User's Guide]((https://www.ntop.org/guides/pf_ring) and access detailed [API Documentation](http://www.ntop.org/guides/pf_ring_api/files.html).
 
-## Provisioning mode
+## Loading PF_RING module on Kubernetes nodes
 
-There are two approaches for loading the PF_RING kernel module on nodes:
+PF_RING kernel module loading is performed via of the `worker` component pod.
+The target container `tap.kernelModule.image` must contain `pf_ring.ko` file under path `/opt/lib/modules/<kernel version>/pf_ring.ko`.
+Kubeshark provides ready to use containers with kernel modules for the most popular kernel versions running in different managed clouds.
 
-1. `auto`
+Prior to deploying `kubeshark` with PF_RING enabled, it is essential to verify if a PF_RING kernel module is already built for your kernel version.
-
-In this mode, the Kubeshark worker retrieves the necessary PF_RING kernel module version from an S3 bucket and loads it onto the node.
-
-> mode=auto requires an active internet connection and is not suitable for air-gapped environments.
-
-2. `kmm`
-
-The Kernel Module Management controller ([KMM](https://kmm.sigs.k8s.io/documentation/deploy_kmod/)) acquires the required PF_RING kernel module version from a Docker container and loads it onto the node
-
-> mode=kmm is suitable for air-gapped environments.
-
-## Selection of Provisioning Mode
-
-> This step is optional. In case mode=auto and no PF_RING kernel module is found Kubeshark falls back to `libpcap` if `PF_RING` kernel module not available
-
-Prior to choosing a method, it is essential to verify if a PF_RING kernel module is already built for your kernel version.
 Kubeshark provides additional CLI tool for this purpose - [pf-ring-compiler](https://github.com/kubeshark/pf-ring-compiler).
 
 Compatibility verification can be done by running:
 
-```
+```bash
 pfring-compiler compatibility
 ```
 
 This command checks for the availability of kernel modules for the kernel versions running across all nodes in the Kubernetes cluster.
 
 Example output for a compatible cluster:
-```
+
+```bash
 Node                                          Kernel Version                 Supported
 ip-192-168-77-230.us-west-2.compute.internal  5.10.199-190.747.amzn2.x86_64  true
 ip-192-168-34-216.us-west-2.compute.internal  5.10.199-190.747.amzn2.x86_64  true
@@ -61,66 +46,41 @@ ip-192-168-34-216.us-west-2.compute.internal  5.10.199-190.747.amzn2.x86_64  tru
 Cluster is compatible
 ```
 
+Another option to verify availability of kernel modules is just inspecting available kernel module versions via:
+
+```bash
+curl https://api.kubeshark.co/kernel-modules/meta/versions.jso
+```
+
+Based on Kubernetes cluster compatibility and external connection capabilities, user has two options:
+
+1. Use Kubeshark provided container `kubeshark/pf-ring-module`
+2. Build custom container with required kernel module version. 
 
 ### Pre-built kernel module exists and external egress allowed
 
-If PF_RING kernel modules are already available for the target nodes (cluster is compatible), both `auto` and `kmm` modes are applicable.
+In this case no additional configuration required.
-
+Kubeshark will load PF_RING kernel module from the default `kubeshark/pf-ring-module:all` container.
-|auto|kmm|
-|----|---|
-| `SYS_MODULE` capability required for Kubeshark | `SYS_MODULE` capability is **not** required for Kubeshark|
-| no additional dependencies | (!)requires `cert-manager` and `KMM` installed (follow [instructions](https://kmm.sigs.k8s.io/documentation/install/) or a specific [cloud platform guide](https://kmm.sigs.k8s.io/lab/)) |
-| Kubeshark falls back to `libpcap` if `PF_RING` kernel module not available | Kubshark waits until PF_RING is loaded with KMM|
-| module is downloaded from S3 bucket in AWS | module is loaded from `kubeshark/pf-ring-module:<kernel version>` container|
-| requires egress connectivity to AWS S3 endpoints | can work in an air-gapped environment when the docker images are stored in a local container registry|
-
 
 ### Pre-built kernel module doesn't exist or external egress isn't allowed
 
-In cases where PF_RING kernel modules are not yet available for the target nodes, or if external egress is restricted, the `kmm` mode is the only viable option (`auto` mode would start Kubeshark with libpcap, not PF_RING).
+In this case building custom Docker image is required.
-This approach enables the use of custom container images as the source for PF_RING kernel modules and allows leveraging private container registries.
 
-#### Steps to Use kmm with Custom Containers
+1. Compile PF_RING kernel module for target version
 
-1. Compile the pf_ring.ko kernel module for your target kernel version (see [Appendix B](#appendix-b-pf_ring-kernel-module-compilation) for instructions).
+Skip if you have `pf_ring.ko` for the target kernel version.
+Otherwise, follow [Appendix A](#appendix-a-pf_ring-kernel-module-compilation) for details.
 
-After building the module with kubeshark pfring compile, you will obtain a `pf-ring-<kernel version>.ko` file.
+2. Build container
-If manually built, rename the kernel module to this format.
-
-2. Build and push Docker container(-s) with the kernel module file from stage 1.
-
-Create `Dockerfile` in the folder with PF_RING kernel module:
-
-```
-FROM alpine:3.18
-ARG KERNEL_VERSION
-
-COPY pf-ring-${KERNEL_VERSION}.ko /opt/lib/modules/${KERNEL_VERSION}/pf_ring.ko
-RUN apk add kmod
-
-RUN depmod -b /opt ${KERNEL_VERSION}
-```
-
-Run build&command:
-
-```
-docker build --build-arg <kernel version> <your registry>/<image>:<kernel version>
-docker push <your registry>:/<image>:<kernel version>
-```
-
-It is recommended to use kernel version as a container tag for consistency.
 
+The same build process Kubeshark has can be reused (follow [pfring-compilier](https://github.com/kubeshark/pf-ring-compiler/tree/main/modules) for details).
 
 3. Configure Helm values
 
-```
+```yaml
 tap:
   kernelModule:
-    mode: kmm
+    image: <container from stage 2>
-    kernelMappings:
-    - regexp: '<kernel version>'
-      containerImage: '<your-registry>/<image>:<kernel version>'
-    imageRepoSecret: <optional secret with credentials for private registry>
 ```
 
 
@@ -132,21 +92,24 @@ PF_RING kernel module compilation can be completed automatically or manually.
 
 In case your Kubernetes workers run supported Linux distribution, `kubeshark` CLI can be used to build PF_RING module:
 
-```
+```bash
 pfring-compiler compile --target <distro>
 ```
 
 This command requires:
+
 - kubectl to be installed and configured with a proper context
 - egress connection to Internet available
 
 This command:
+
 1. Runs Kubernetes job with build container
 2. Waits for job to be completed
 3. Downloads `pf-ring-<kernel version>.ko` file into the current folder.
 4. Cleans up created job.
 
 Currently supported distros:
+
 - Ubuntu
 - RHEL 9
 - Amazon Linux 2
@@ -158,13 +121,13 @@ The process description is based on Ubuntu 22.04 distribution.
 1. Get terminal access to the node with target kernel version
 This can be done either via SSH directly to node or with debug container running on the target node:
 
-```
+```bash
 kubectl debug node/<target node> -it --attach=true --image=ubuntu:22.04
 ```
 
 2. Install build tools and kernel headers
 
-```
+```bash
 apt update
 apt install -y gcc build-essential make git wget tar gzip
 apt install -y linux-headers-$(uname -r)
@@ -172,7 +135,7 @@ apt install -y linux-headers-$(uname -r)
 
 3. Download PF_RING source code
 
-```
+```bash
 wget https://github.com/ntop/PF_RING/archive/refs/tags/8.4.0.tar.gz
 tar -xf 8.4.0.tar.gz
 cd PF_RING-8.4.0/kernel
@@ -180,7 +143,7 @@ cd PF_RING-8.4.0/kernel
 
 4. Compile the kernel module
 
-```
+```bash
 make KERNEL_SRC=/usr/src/linux-headers-$(uname -r)
 ```
 

helm-chart/README.md

@@ -156,10 +156,9 @@ Please refer to [metrics](./metrics.md) documentation for details.
 | `tap.ingress.annotations`                 | `Ingress` annotations                           | `{}`                                                    |
 | `tap.ipv6`                                | Enable IPv6 support for the front-end                        | `true`                                                  |
 | `tap.debug`                               | Enable debug mode                             | `false`                                                 |
-| `tap.kernelModule.enabled`                    | Use PF_RING kernel module([details](PF_RING.md))      | `true`                                                 |
+| `tap.kernelModule.enabled`                | Use PF_RING kernel module([details](PF_RING.md))      | `true`                                                 |
-| `tap.kernelModule.mode`                    | PF_RING kernel module loading approach([details](PF_RING.md))      | `auto`                                                 |
+| `tap.kernelModule.image`                  | Container image containing PF_RING kernel module with supported kernel version([details](PF_RING.md))      | "kubeshark/pf-ring-module:all"                                                 |
-| `tap.kernelModule.imageRepoSecret`                    | ImageRepoSecret is an optional secret that is used to pull both the module loader container([details](PF_RING.md))      | ""                                                 |
+| `tap.kernelModule.unloadOnDestroy`        | Create additional container which watches for pod termination and unloads PF_RING kernel module. | `false`|
-| `tap.kernelModule.kernelMappings`                    |List of mappings between kernel version and container loader([details](PF_RING.md))      | `[{'regexp': '.+$', 'containerImage': 'kubehq/pf-ring-module:${KERNEL_FULL_VERSION}'}]`                                                 |
 | `tap.telemetry.enabled`                   | Enable anonymous usage statistics collection           | `true`                                                  |
 | `tap.defaultFilter`                       | Sets the default dashboard KFL filter (e.g. `http`)        | `""`                                                  |
 | `tap.globalFilter`                        | Prepends to any KFL filter and can be used to limit what is visible in the dashboard. For example, `redact("request.headers.Authorization")` will redact the appropriate field.       | `""`                                        |

helm-chart/templates/09-worker-daemon-set.yaml

@@ -25,22 +25,22 @@ spec:
       name: kubeshark-worker-daemon-set
       namespace: kubeshark
     spec:
-    {{- if and (eq .Values.tap.kernelModule.enabled true) (eq .Values.tap.kernelModule.mode "kmm")}}
+    {{- if .Values.tap.kernelModule.enabled }}
       initContainers:
-      - name: wait-for-pf-ring
+      - name: load-pf-ring
-        image: alpine:3.18
+        image: {{ .Values.tap.kernelModule.image }}
-        command: ["/bin/sh", "-c"]
+        imagePullPolicy: {{ .Values.tap.docker.imagePullPolicy }}
-        args:
+        securityContext:
-        - >
+          capabilities:
-          while true; do
+            add:
-            if lsmod | grep -q "pf_ring"; then
+              {{- range .Values.tap.capabilities.kernelModule }}
-              echo "pf_ring module is loaded.";
+              {{ print "- " . }}
-              break;
+              {{- end }}
-            else
+            drop:
-              echo "Waiting for pf_ring module to be loaded...";
+              - ALL
-              sleep 5;
+        volumeMounts:
-            fi
+        - name: lib-modules
-          done
+          mountPath: /lib/modules
     {{- end }}
       containers:
         - command:
@@ -56,9 +56,8 @@ spec:
           {{- end }}
             - -procfs
             - /hostproc
-          {{- if or (eq .Values.tap.kernelModule.enabled false) (eq .Values.tap.kernelModule.mode "kmm") }}
+            # TODO: remove
             - -no-kernel-module
-          {{ end }}
           {{- if .Values.tap.debug }}
             - -debug
           {{- end }}
@@ -91,11 +90,6 @@ spec:
                 {{- range .Values.tap.capabilities.networkCapture }}
                 {{ print "- " . }}
                 {{- end }}
-                {{- if and (.Values.tap.kernelModule.enabled) (eq .Values.tap.kernelModule.mode "auto") }}
-                {{- range .Values.tap.capabilities.kernelModule }}
-                {{ print "- " . }}
-                {{- end }}
-                {{- end }}
                 {{- if .Values.tap.serviceMesh }}
                 {{- range .Values.tap.capabilities.serviceMeshCapture }}
                 {{ print "- " . }}
@@ -126,6 +120,20 @@ spec:
               readOnly: true
             - mountPath: /app/data
               name: data
+      {{- if and (eq .Values.tap.kernelModule.enabled true) (eq .Values.tap.kernelModule.unloadOnDestroy true) }}
+        - name: unload-pf-ring
+          image: {{ .Values.tap.kernelModule.image }}
+          command: ["/bin/sh"]
+          args: ["-c", "trap 'rmmod pf_ring && sleep 3' SIGTERM; while true; do sleep 1; done"] 
+          securityContext:
+            capabilities:
+              add:
+                {{- range .Values.tap.capabilities.kernelModule }}
+                {{ print "- " . }}
+                {{- end }}
+              drop:
+                - ALL
+      {{- end }}
       {{- if .Values.tap.tls }}
         - command:
             - ./tracer
@@ -196,6 +204,9 @@ spec:
         - hostPath:
             path: /sys
           name: sys
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
         - name: data
 {{- if .Values.tap.persistentStorage }}
           persistentVolumeClaim:

helm-chart/templates/15-pf-ring-kernel-module.yaml

@@ -1,84 +0,0 @@
----
-{{- if .Values.tap.kernelModule.enabled }}
-{{- if not (or (eq .Values.tap.kernelModule.mode "auto") (eq .Values.tap.kernelModule.mode "kmm")) }}
-{{- fail "Invalid value for tap.kernelModule.mode; must be 'auto' or 'kmm'" }}
-{{- end }}
-{{- if eq .Values.tap.kernelModule.mode "kmm" }}
-apiVersion: kmm.sigs.x-k8s.io/v1beta1
-kind: Module
-metadata:
-  name: pf-ring
-spec:
-  moduleLoader:
-    container:
-      modprobe:
-        moduleName: pf_ring
-        dirName: /opt
-      kernelMappings:
-      -
-      {{- range .Values.tap.kernelModule.kernelMappings }}
-      {{- toYaml . | nindent 8 }}
-      {{- end }}
-  selector:
-    kubernetes.io/hostname:
-  {{- if ne .Values.tap.kernelModule.imageRepoSecret "" }}
-  imageRepoSecret:
-    name: {{ .Values.tap.kernelModule.imageRepoSecret }}
-  {{- end }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: pfring-kmm-status-check-script
-data:
-  check_status.sh: |
-    #!/bin/bash
-
-    timeout=300
-    interval=10
-    elapsed=0
-
-    while [ $elapsed -lt $timeout ]; do
-        desired=$(kubectl get modules pf-ring -o json | jq -r .status.moduleLoader.desiredNumber)
-        available=$(kubectl get modules pf-ring -o json | jq -r .status.moduleLoader.availableNumber)
-
-        echo "Checking desired and available module load: $desired vs $available"
-
-        if [ "$desired" = "$available" ]; then
-            echo "PF_RING loaded an all nodes."
-            exit 0
-        else
-            sleep $interval
-            elapsed=$((elapsed + interval))
-        fi
-    done
-
-    echo "Timeout reached. PF_RING module didn't reach desired status."
-    exit 1
-
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: pfring-kmm-status-check
-  annotations:
-    "helm.sh/hook": post-install,post-upgrade
-    "helm.sh/hook-weight": "1"
-    "helm.sh/hook-delete-policy": hook-succeeded
-spec:
-  template:
-    spec:
-      containers:
-      - name: verify-pfring-kmm-status
-        image: alpine/k8s:1.27.8
-        command: ["/bin/bash", "/scripts/check_status.sh"]
-        volumeMounts:
-        - name: script-volume
-          mountPath: /scripts
-      volumes:
-      - name: script-volume
-        configMap:
-          name: pfring-kmm-status-check-script
-      restartPolicy: Never
-{{- end }}
-{{- end }}

helm-chart/values.yaml

@@ -32,14 +32,14 @@ tap:
       requests:
         cpu: 50m
         memory: 50Mi
-    tracer:
+    hub:
       limits:
         cpu: 750m
         memory: 1Gi
       requests:
         cpu: 50m
-        memory: 150Mi
+        memory: 50Mi
-    hub:
+    tracer:
       limits:
         cpu: 750m
         memory: 1Gi
@@ -72,11 +72,8 @@ tap:
   debug: false
   kernelModule:
     enabled: true
-    mode: auto
+    image: kubeshark/pf-ring-module:all
-    kernelMappings:
+    unloadOnDestroy: false
-    - regexp: ^.+$
-      containerImage: kubeshark/pf-ring-module:${KERNEL_FULL_VERSION}
-    imageRepoSecret: ""
   telemetry:
     enabled: true
   defaultFilter: ""