✨ Add kubeshark-worker-metrics service and document it (#1474)

* Expose worker metrics

* Add metrics documentation

* upd

* Update metrics port configuration

* Update config/configStructs/tapConfig.go

Co-authored-by: M. Mert Yildiran <me@mertyildiran.com>

* Update helm-chart/README.md

Co-authored-by: M. Mert Yildiran <me@mertyildiran.com>

* Update helm-chart/templates/16-worker-service-metrics.yaml

Co-authored-by: M. Mert Yildiran <me@mertyildiran.com>

---------

Co-authored-by: M. Mert Yildiran <me@mertyildiran.com>

config/configStructs/tapConfig.go

@@ -125,6 +125,10 @@ type KernelModuleConfig struct {
 	ImageRepoSecret string          `yaml:"imageRepoSecret" json:"imageRepoSecret"`
 }
 
+type MetricsConfig struct {
+	Port uint16 `yaml:"port" json:"port" default:"49100"`
+}
+
 type TapConfig struct {
 	Docker                  DockerConfig          `yaml:"docker" json:"docker"`
 	Proxy                   ProxyConfig           `yaml:"proxy" json:"proxy"`
@@ -154,6 +158,7 @@ type TapConfig struct {
 	ReplayDisabled          bool                  `yaml:"replayDisabled" json:"replayDisabled" default:"false"`
 	Capabilities            CapabilitiesConfig    `yaml:"capabilities" json:"capabilities"`
 	GlobalFilter            string                `yaml:"globalFilter" json:"globalFilter"`
+	Metrics                 MetricsConfig         `yaml:"metrics" json:"metrics"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {

helm-chart/README.md

@@ -104,6 +104,10 @@ helm install kubeshark kubeshark/kubeshark \
   --set tap.ipv6=false
 ```
 
+## Metrics
+
+Please refer to [metrics](./metrics.md) documentation for details.
+
 ## Configuration
 
 | Parameter                                 | Description                                   | Default                                                 |
@@ -168,6 +172,7 @@ helm install kubeshark kubeshark/kubeshark \
 | `scripting.env`                           | Environment variables for the scripting      | `{}`                                                    |
 | `scripting.source`                        | Source directory of the scripts                | `""`                                                    |
 | `scripting.watchScripts`                  | Enable watch mode for the scripts in source directory          | `true`                                                  |
+| `tap.metrics.port`                  | Pod port used to expose Prometheus metrics          | `49100`                                                  |
 
 KernelMapping pairs kernel versions with a
                             DriverContainer image. Kernel versions can be matched

helm-chart/metrics.md

@@ -0,0 +1,51 @@
+# Metrics
+
+Kubeshark provides metrics from `worker` components.
+It can be useful for monitoring and debugging purpose.
+
+## Configuration
+
+By default, Kubeshark uses port `49100` to expose metrics via service `kubeshark-worker-metrics`.
+
+In case you use [kube-prometheus-stack] (https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) community Helm chart, additional scrape configuration for Kubeshark worker metrics endpoint can be configured with values:
+
+```
+prometheus:
+  enabled: true
+  prometheusSpec:
+    additionalScrapeConfigs: |
+      - job_name: 'kubeshark-worker-metrics'
+        kubernetes_sd_configs:
+          - role: endpoints
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_pod_name]
+            target_label: pod
+          - source_labels: [__meta_kubernetes_pod_node_name]
+            target_label: node
+          - source_labels: [__meta_kubernetes_endpoint_port_name]
+            action: keep
+            regex: ^metrics$
+          - source_labels: [__address__, __meta_kubernetes_endpoint_port_number]
+            action: replace
+            regex: ([^:]+)(?::\d+)?
+            replacement: $1:49100
+            target_label: __address__
+          - action: labelmap
+            regex: __meta_kubernetes_service_label_(.+)
+```
+
+
+## Available metrics
+
+| Name | Type | Description | 
+| --- | --- | --- | 
+| kubeshark_received_packets_total | Counter | Total number of packets received | 
+| kubeshark_dropped_packets_total | Counter | Total number of packets dropped | 
+| kubeshark_processed_bytes_total | Counter | Total number of bytes processed |
+| kubeshark_tcp_packets_total | Counter | Total number of TCP packets | 
+| kubeshark_dns_packets_total | Counter | Total number of DNS packets | 
+| kubeshark_icmp_packets_total | Counter | Total number of ICMP packets | 
+| kubeshark_reassembled_tcp_payloads_total | Counter | Total number of reassembled TCP payloads |
+| kubeshark_matched_pairs_total | Counter | Total number of matched pairs | 
+| kubeshark_dropped_tcp_streams_total | Counter | Total number of dropped TCP streams | 
+| kubeshark_live_tcp_streams | Gauge | Number of live TCP streams |

helm-chart/templates/09-worker-daemon-set.yaml

@@ -49,6 +49,8 @@ spec:
             - any
             - -port
             - '{{ .Values.tap.proxy.worker.srvPort }}'
+            - -metrics-port
+            - '{{ .Values.tap.metrics.port }}'
           {{- if .Values.tap.serviceMesh }}
             - -servicemesh
           {{- end }}
@@ -63,6 +65,10 @@ spec:
           image: '{{ .Values.tap.docker.registry }}/worker:{{ not (eq .Values.tap.docker.tag "") | ternary .Values.tap.docker.tag (printf "v%s" .Chart.Version) }}'
           imagePullPolicy: {{ .Values.tap.docker.imagePullPolicy }}
           name: sniffer
+          ports:
+            - containerPort: {{ .Values.tap.metrics.port }}
+              protocol: TCP
+              name: metrics
           env:
           - name: POD_NAME
             valueFrom:

helm-chart/templates/16-worker-service-metrics.yaml

@@ -0,0 +1,18 @@
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kubeshark-worker-metrics
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    prometheus.io/scrape: 'true'
+    prometheus.io/port: '{{ .Values.tap.metrics.port }}'
+spec:
+  selector:
+    app.kubeshark.co/app: worker
+    {{- include "kubeshark.labels" . | nindent 4 }}
+  ports:
+  - name: metrics
+    protocol: TCP
+    port: {{ .Values.tap.metrics.port }}
+    targetPort: {{ .Values.tap.metrics.port }}

helm-chart/values.yaml

@@ -91,6 +91,8 @@ tap:
     - SYS_RESOURCE
     - CHECKPOINT_RESTORE
   globalFilter: ""
+  metrics:
+    port: 49100
 logs:
   file: ""
 kube: