From aaeb3ca1eb8d5cfa512fc1f7bc07c90fbb662317 Mon Sep 17 00:00:00 2001 From: Volodymyr Stoiko Date: Sat, 13 Jan 2024 01:49:39 +0200 Subject: [PATCH] Load pf-ring kernel module in init container (#1476) * Load kernel module in init container * Update docs * Update formatting * Add pre-stop hook to unload pf_ring module * Enable hook only on kernel module enabled * fix template * Use sidecontainer to unload pf_ring * Add requirements for tracer into structs * fix values * fix typo --------- Co-authored-by: Alon Girmonsky <1990761+alongir@users.noreply.github.com> --- config/configStruct.go | 8 -- config/configStructs/tapConfig.go | 13 +- helm-chart/PF_RING.md | 125 ++++++------------ helm-chart/README.md | 7 +- .../templates/09-worker-daemon-set.yaml | 55 +++++--- .../templates/15-pf-ring-kernel-module.yaml | 84 ------------ ...cs.yaml => 15-worker-service-metrics.yaml} | 0 helm-chart/values.yaml | 13 +- 8 files changed, 89 insertions(+), 216 deletions(-) delete mode 100644 helm-chart/templates/15-pf-ring-kernel-module.yaml rename helm-chart/templates/{16-worker-service-metrics.yaml => 15-worker-service-metrics.yaml} (100%) diff --git a/config/configStruct.go b/config/configStruct.go index 57133952f..7666f5fde 100644 --- a/config/configStruct.go +++ b/config/configStruct.go @@ -16,14 +16,6 @@ const ( func CreateDefaultConfig() ConfigStruct { return ConfigStruct{ Tap: configStructs.TapConfig{ - KernelModule: configStructs.KernelModuleConfig{ - KernelMappings: []configStructs.KernelMapping{ - { - ContainerImage: "kubeshark/pf-ring-module:${KERNEL_FULL_VERSION}", - Regexp: "^.+$", - }, - }, - }, NodeSelectorTerms: []v1.NodeSelectorTerm{ { MatchExpressions: []v1.NodeSelectorRequirement{ diff --git a/config/configStructs/tapConfig.go b/config/configStructs/tapConfig.go index cb8554df7..2b10c2359 100644 --- a/config/configStructs/tapConfig.go +++ b/config/configStructs/tapConfig.go @@ -79,6 +79,7 @@ type DockerConfig struct { type ResourcesConfig struct { Worker ResourceRequirements `yaml:"worker" json:"worker"` Hub ResourceRequirements `yaml:"hub" json:"hub"` + Tracer ResourceRequirements `yaml:"tracer" json:"tracer"` } type AuthConfig struct { @@ -113,16 +114,10 @@ type CapabilitiesConfig struct { EBPFCapture []string `yaml:"ebpfCapture" json:"ebpfCapture" default:"[]"` } -type KernelMapping struct { - Regexp string `yaml:"regexp" json:"regexp"` - ContainerImage string `yaml:"containerImage" json:"containerImage"` -} - type KernelModuleConfig struct { - Enabled bool `yaml:"enabled" json:"enabled" default:"true"` - Mode string `yaml:"mode" json:"mode" default:"auto"` - KernelMappings []KernelMapping `yaml:"kernelMappings" json:"kernelMappings"` - ImageRepoSecret string `yaml:"imageRepoSecret" json:"imageRepoSecret"` + Enabled bool `yaml:"enabled" json:"enabled" default:"true"` + Image string `yaml:"image" json:"image" default:"kubeshark/pf-ring-module:all"` + UnloadOnDestroy bool `yaml:"unloadOnDestroy" json:"unloadOnDestroy" default:"false"` } type MetricsConfig struct { diff --git a/helm-chart/PF_RING.md b/helm-chart/PF_RING.md index bae39df34..9d60db79b 100644 --- a/helm-chart/PF_RING.md +++ b/helm-chart/PF_RING.md @@ -2,16 +2,14 @@ -- [PF_RING](#pf_ring) - - [Overview](#overview) - - [Provisioning mode](#provisioning-mode) - - [Selection of Provisioning Mode](#selection-of-provisioning-mode) - - [Pre-built kernel module exists and external egress allowed](#pre-built-kernel-module-exists-and-external-egress-allowed) - - [Pre-built kernel module doesn't exist or external egress isn't allowed](#pre-built-kernel-module-doesnt-exist-or-external-egress-isnt-allowed) - - [Steps to Use kmm with Custom Containers](#steps-to-use-kmm-with-custom-containers) - - [Appendix A: PF_RING kernel module compilation](#appendix-a-pf_ring-kernel-module-compilation) - - [Automated complilation](#automated-complilation) - - [Manual compilation](#manual-compilation) +- [PF\_RING](#pf_ring) + - [Overview](#overview) + - [Loading PF\_RING module on Kubernetes nodes](#loading-pf_ring-module-on-kubernetes-nodes) + - [Pre-built kernel module exists and external egress allowed](#pre-built-kernel-module-exists-and-external-egress-allowed) + - [Pre-built kernel module doesn't exist or external egress isn't allowed](#pre-built-kernel-module-doesnt-exist-or-external-egress-isnt-allowed) + - [Appendix A: PF\_RING kernel module compilation](#appendix-a-pf_ring-kernel-module-compilation) + - [Automated complilation](#automated-complilation) + - [Manual compilation](#manual-compilation) @@ -21,39 +19,26 @@ PF_RING™ is an advanced Linux kernel module and user-space framework designed For comprehensive information on PF_RING™, please visit the [User's Guide]((https://www.ntop.org/guides/pf_ring) and access detailed [API Documentation](http://www.ntop.org/guides/pf_ring_api/files.html). -## Provisioning mode +## Loading PF_RING module on Kubernetes nodes -There are two approaches for loading the PF_RING kernel module on nodes: +PF_RING kernel module loading is performed via of the `worker` component pod. +The target container `tap.kernelModule.image` must contain `pf_ring.ko` file under path `/opt/lib/modules//pf_ring.ko`. +Kubeshark provides ready to use containers with kernel modules for the most popular kernel versions running in different managed clouds. -1. `auto` - -In this mode, the Kubeshark worker retrieves the necessary PF_RING kernel module version from an S3 bucket and loads it onto the node. - -> mode=auto requires an active internet connection and is not suitable for air-gapped environments. - -2. `kmm` - -The Kernel Module Management controller ([KMM](https://kmm.sigs.k8s.io/documentation/deploy_kmod/)) acquires the required PF_RING kernel module version from a Docker container and loads it onto the node - -> mode=kmm is suitable for air-gapped environments. - -## Selection of Provisioning Mode - -> This step is optional. In case mode=auto and no PF_RING kernel module is found Kubeshark falls back to `libpcap` if `PF_RING` kernel module not available - -Prior to choosing a method, it is essential to verify if a PF_RING kernel module is already built for your kernel version. +Prior to deploying `kubeshark` with PF_RING enabled, it is essential to verify if a PF_RING kernel module is already built for your kernel version. Kubeshark provides additional CLI tool for this purpose - [pf-ring-compiler](https://github.com/kubeshark/pf-ring-compiler). Compatibility verification can be done by running: -``` +```bash pfring-compiler compatibility ``` This command checks for the availability of kernel modules for the kernel versions running across all nodes in the Kubernetes cluster. Example output for a compatible cluster: -``` + +```bash Node Kernel Version Supported ip-192-168-77-230.us-west-2.compute.internal 5.10.199-190.747.amzn2.x86_64 true ip-192-168-34-216.us-west-2.compute.internal 5.10.199-190.747.amzn2.x86_64 true @@ -61,66 +46,41 @@ ip-192-168-34-216.us-west-2.compute.internal 5.10.199-190.747.amzn2.x86_64 tru Cluster is compatible ``` +Another option to verify availability of kernel modules is just inspecting available kernel module versions via: + +```bash +curl https://api.kubeshark.co/kernel-modules/meta/versions.jso +``` + +Based on Kubernetes cluster compatibility and external connection capabilities, user has two options: + +1. Use Kubeshark provided container `kubeshark/pf-ring-module` +2. Build custom container with required kernel module version. ### Pre-built kernel module exists and external egress allowed -If PF_RING kernel modules are already available for the target nodes (cluster is compatible), both `auto` and `kmm` modes are applicable. - -|auto|kmm| -|----|---| -| `SYS_MODULE` capability required for Kubeshark | `SYS_MODULE` capability is **not** required for Kubeshark| -| no additional dependencies | (!)requires `cert-manager` and `KMM` installed (follow [instructions](https://kmm.sigs.k8s.io/documentation/install/) or a specific [cloud platform guide](https://kmm.sigs.k8s.io/lab/)) | -| Kubeshark falls back to `libpcap` if `PF_RING` kernel module not available | Kubshark waits until PF_RING is loaded with KMM| -| module is downloaded from S3 bucket in AWS | module is loaded from `kubeshark/pf-ring-module:` container| -| requires egress connectivity to AWS S3 endpoints | can work in an air-gapped environment when the docker images are stored in a local container registry| - +In this case no additional configuration required. +Kubeshark will load PF_RING kernel module from the default `kubeshark/pf-ring-module:all` container. ### Pre-built kernel module doesn't exist or external egress isn't allowed -In cases where PF_RING kernel modules are not yet available for the target nodes, or if external egress is restricted, the `kmm` mode is the only viable option (`auto` mode would start Kubeshark with libpcap, not PF_RING). -This approach enables the use of custom container images as the source for PF_RING kernel modules and allows leveraging private container registries. +In this case building custom Docker image is required. -#### Steps to Use kmm with Custom Containers +1. Compile PF_RING kernel module for target version -1. Compile the pf_ring.ko kernel module for your target kernel version (see [Appendix B](#appendix-b-pf_ring-kernel-module-compilation) for instructions). +Skip if you have `pf_ring.ko` for the target kernel version. +Otherwise, follow [Appendix A](#appendix-a-pf_ring-kernel-module-compilation) for details. -After building the module with kubeshark pfring compile, you will obtain a `pf-ring-.ko` file. -If manually built, rename the kernel module to this format. - -2. Build and push Docker container(-s) with the kernel module file from stage 1. - -Create `Dockerfile` in the folder with PF_RING kernel module: - -``` -FROM alpine:3.18 -ARG KERNEL_VERSION - -COPY pf-ring-${KERNEL_VERSION}.ko /opt/lib/modules/${KERNEL_VERSION}/pf_ring.ko -RUN apk add kmod - -RUN depmod -b /opt ${KERNEL_VERSION} -``` - -Run build&command: - -``` -docker build --build-arg /: -docker push :/: -``` - -It is recommended to use kernel version as a container tag for consistency. +2. Build container +The same build process Kubeshark has can be reused (follow [pfring-compilier](https://github.com/kubeshark/pf-ring-compiler/tree/main/modules) for details). 3. Configure Helm values -``` +```yaml tap: kernelModule: - mode: kmm - kernelMappings: - - regexp: '' - containerImage: '/:' - imageRepoSecret: + image: ``` @@ -132,21 +92,24 @@ PF_RING kernel module compilation can be completed automatically or manually. In case your Kubernetes workers run supported Linux distribution, `kubeshark` CLI can be used to build PF_RING module: -``` +```bash pfring-compiler compile --target ``` This command requires: + - kubectl to be installed and configured with a proper context - egress connection to Internet available This command: + 1. Runs Kubernetes job with build container 2. Waits for job to be completed 3. Downloads `pf-ring-.ko` file into the current folder. 4. Cleans up created job. Currently supported distros: + - Ubuntu - RHEL 9 - Amazon Linux 2 @@ -158,13 +121,13 @@ The process description is based on Ubuntu 22.04 distribution. 1. Get terminal access to the node with target kernel version This can be done either via SSH directly to node or with debug container running on the target node: -``` +```bash kubectl debug node/ -it --attach=true --image=ubuntu:22.04 ``` 2. Install build tools and kernel headers -``` +```bash apt update apt install -y gcc build-essential make git wget tar gzip apt install -y linux-headers-$(uname -r) @@ -172,7 +135,7 @@ apt install -y linux-headers-$(uname -r) 3. Download PF_RING source code -``` +```bash wget https://github.com/ntop/PF_RING/archive/refs/tags/8.4.0.tar.gz tar -xf 8.4.0.tar.gz cd PF_RING-8.4.0/kernel @@ -180,7 +143,7 @@ cd PF_RING-8.4.0/kernel 4. Compile the kernel module -``` +```bash make KERNEL_SRC=/usr/src/linux-headers-$(uname -r) ``` diff --git a/helm-chart/README.md b/helm-chart/README.md index a25a27039..26babfa47 100644 --- a/helm-chart/README.md +++ b/helm-chart/README.md @@ -156,10 +156,9 @@ Please refer to [metrics](./metrics.md) documentation for details. | `tap.ingress.annotations` | `Ingress` annotations | `{}` | | `tap.ipv6` | Enable IPv6 support for the front-end | `true` | | `tap.debug` | Enable debug mode | `false` | -| `tap.kernelModule.enabled` | Use PF_RING kernel module([details](PF_RING.md)) | `true` | -| `tap.kernelModule.mode` | PF_RING kernel module loading approach([details](PF_RING.md)) | `auto` | -| `tap.kernelModule.imageRepoSecret` | ImageRepoSecret is an optional secret that is used to pull both the module loader container([details](PF_RING.md)) | "" | -| `tap.kernelModule.kernelMappings` |List of mappings between kernel version and container loader([details](PF_RING.md)) | `[{'regexp': '.+$', 'containerImage': 'kubehq/pf-ring-module:${KERNEL_FULL_VERSION}'}]` | +| `tap.kernelModule.enabled` | Use PF_RING kernel module([details](PF_RING.md)) | `true` | +| `tap.kernelModule.image` | Container image containing PF_RING kernel module with supported kernel version([details](PF_RING.md)) | "kubeshark/pf-ring-module:all" | +| `tap.kernelModule.unloadOnDestroy` | Create additional container which watches for pod termination and unloads PF_RING kernel module. | `false`| | `tap.telemetry.enabled` | Enable anonymous usage statistics collection | `true` | | `tap.defaultFilter` | Sets the default dashboard KFL filter (e.g. `http`) | `""` | | `tap.globalFilter` | Prepends to any KFL filter and can be used to limit what is visible in the dashboard. For example, `redact("request.headers.Authorization")` will redact the appropriate field. | `""` | diff --git a/helm-chart/templates/09-worker-daemon-set.yaml b/helm-chart/templates/09-worker-daemon-set.yaml index 062de7576..702f08845 100644 --- a/helm-chart/templates/09-worker-daemon-set.yaml +++ b/helm-chart/templates/09-worker-daemon-set.yaml @@ -25,22 +25,22 @@ spec: name: kubeshark-worker-daemon-set namespace: kubeshark spec: - {{- if and (eq .Values.tap.kernelModule.enabled true) (eq .Values.tap.kernelModule.mode "kmm")}} + {{- if .Values.tap.kernelModule.enabled }} initContainers: - - name: wait-for-pf-ring - image: alpine:3.18 - command: ["/bin/sh", "-c"] - args: - - > - while true; do - if lsmod | grep -q "pf_ring"; then - echo "pf_ring module is loaded."; - break; - else - echo "Waiting for pf_ring module to be loaded..."; - sleep 5; - fi - done + - name: load-pf-ring + image: {{ .Values.tap.kernelModule.image }} + imagePullPolicy: {{ .Values.tap.docker.imagePullPolicy }} + securityContext: + capabilities: + add: + {{- range .Values.tap.capabilities.kernelModule }} + {{ print "- " . }} + {{- end }} + drop: + - ALL + volumeMounts: + - name: lib-modules + mountPath: /lib/modules {{- end }} containers: - command: @@ -56,9 +56,8 @@ spec: {{- end }} - -procfs - /hostproc - {{- if or (eq .Values.tap.kernelModule.enabled false) (eq .Values.tap.kernelModule.mode "kmm") }} + # TODO: remove - -no-kernel-module - {{ end }} {{- if .Values.tap.debug }} - -debug {{- end }} @@ -91,11 +90,6 @@ spec: {{- range .Values.tap.capabilities.networkCapture }} {{ print "- " . }} {{- end }} - {{- if and (.Values.tap.kernelModule.enabled) (eq .Values.tap.kernelModule.mode "auto") }} - {{- range .Values.tap.capabilities.kernelModule }} - {{ print "- " . }} - {{- end }} - {{- end }} {{- if .Values.tap.serviceMesh }} {{- range .Values.tap.capabilities.serviceMeshCapture }} {{ print "- " . }} @@ -126,6 +120,20 @@ spec: readOnly: true - mountPath: /app/data name: data + {{- if and (eq .Values.tap.kernelModule.enabled true) (eq .Values.tap.kernelModule.unloadOnDestroy true) }} + - name: unload-pf-ring + image: {{ .Values.tap.kernelModule.image }} + command: ["/bin/sh"] + args: ["-c", "trap 'rmmod pf_ring && sleep 3' SIGTERM; while true; do sleep 1; done"] + securityContext: + capabilities: + add: + {{- range .Values.tap.capabilities.kernelModule }} + {{ print "- " . }} + {{- end }} + drop: + - ALL + {{- end }} {{- if .Values.tap.tls }} - command: - ./tracer @@ -196,6 +204,9 @@ spec: - hostPath: path: /sys name: sys + - name: lib-modules + hostPath: + path: /lib/modules - name: data {{- if .Values.tap.persistentStorage }} persistentVolumeClaim: diff --git a/helm-chart/templates/15-pf-ring-kernel-module.yaml b/helm-chart/templates/15-pf-ring-kernel-module.yaml deleted file mode 100644 index ae53a4609..000000000 --- a/helm-chart/templates/15-pf-ring-kernel-module.yaml +++ /dev/null @@ -1,84 +0,0 @@ ---- -{{- if .Values.tap.kernelModule.enabled }} -{{- if not (or (eq .Values.tap.kernelModule.mode "auto") (eq .Values.tap.kernelModule.mode "kmm")) }} -{{- fail "Invalid value for tap.kernelModule.mode; must be 'auto' or 'kmm'" }} -{{- end }} -{{- if eq .Values.tap.kernelModule.mode "kmm" }} -apiVersion: kmm.sigs.x-k8s.io/v1beta1 -kind: Module -metadata: - name: pf-ring -spec: - moduleLoader: - container: - modprobe: - moduleName: pf_ring - dirName: /opt - kernelMappings: - - - {{- range .Values.tap.kernelModule.kernelMappings }} - {{- toYaml . | nindent 8 }} - {{- end }} - selector: - kubernetes.io/hostname: - {{- if ne .Values.tap.kernelModule.imageRepoSecret "" }} - imageRepoSecret: - name: {{ .Values.tap.kernelModule.imageRepoSecret }} - {{- end }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: pfring-kmm-status-check-script -data: - check_status.sh: | - #!/bin/bash - - timeout=300 - interval=10 - elapsed=0 - - while [ $elapsed -lt $timeout ]; do - desired=$(kubectl get modules pf-ring -o json | jq -r .status.moduleLoader.desiredNumber) - available=$(kubectl get modules pf-ring -o json | jq -r .status.moduleLoader.availableNumber) - - echo "Checking desired and available module load: $desired vs $available" - - if [ "$desired" = "$available" ]; then - echo "PF_RING loaded an all nodes." - exit 0 - else - sleep $interval - elapsed=$((elapsed + interval)) - fi - done - - echo "Timeout reached. PF_RING module didn't reach desired status." - exit 1 - ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: pfring-kmm-status-check - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-weight": "1" - "helm.sh/hook-delete-policy": hook-succeeded -spec: - template: - spec: - containers: - - name: verify-pfring-kmm-status - image: alpine/k8s:1.27.8 - command: ["/bin/bash", "/scripts/check_status.sh"] - volumeMounts: - - name: script-volume - mountPath: /scripts - volumes: - - name: script-volume - configMap: - name: pfring-kmm-status-check-script - restartPolicy: Never -{{- end }} -{{- end }} diff --git a/helm-chart/templates/16-worker-service-metrics.yaml b/helm-chart/templates/15-worker-service-metrics.yaml similarity index 100% rename from helm-chart/templates/16-worker-service-metrics.yaml rename to helm-chart/templates/15-worker-service-metrics.yaml diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 749b352b1..ddaa52139 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -32,14 +32,14 @@ tap: requests: cpu: 50m memory: 50Mi - tracer: + hub: limits: cpu: 750m memory: 1Gi requests: cpu: 50m - memory: 150Mi - hub: + memory: 50Mi + tracer: limits: cpu: 750m memory: 1Gi @@ -72,11 +72,8 @@ tap: debug: false kernelModule: enabled: true - mode: auto - kernelMappings: - - regexp: ^.+$ - containerImage: kubeshark/pf-ring-module:${KERNEL_FULL_VERSION} - imageRepoSecret: "" + image: kubeshark/pf-ring-module:all + unloadOnDestroy: false telemetry: enabled: true defaultFilter: ""