From db51e6dbc23d6c44abf6c0aff88aa7b8d3e92068 Mon Sep 17 00:00:00 2001 From: Volodymyr Stoiko Date: Thu, 4 Jan 2024 15:17:22 +0200 Subject: [PATCH] :sparkles: Add `kubeshark-worker-metrics` service and document it (#1474) * Expose worker metrics * Add metrics documentation * upd * Update metrics port configuration * Update config/configStructs/tapConfig.go Co-authored-by: M. Mert Yildiran * Update helm-chart/README.md Co-authored-by: M. Mert Yildiran * Update helm-chart/templates/16-worker-service-metrics.yaml Co-authored-by: M. Mert Yildiran --------- Co-authored-by: M. Mert Yildiran --- config/configStructs/tapConfig.go | 5 ++ helm-chart/README.md | 5 ++ helm-chart/metrics.md | 51 +++++++++++++++++++ .../templates/09-worker-daemon-set.yaml | 6 +++ .../templates/16-worker-service-metrics.yaml | 18 +++++++ helm-chart/values.yaml | 2 + 6 files changed, 87 insertions(+) create mode 100644 helm-chart/metrics.md create mode 100644 helm-chart/templates/16-worker-service-metrics.yaml diff --git a/config/configStructs/tapConfig.go b/config/configStructs/tapConfig.go index d98098b5d..c460d4952 100644 --- a/config/configStructs/tapConfig.go +++ b/config/configStructs/tapConfig.go @@ -125,6 +125,10 @@ type KernelModuleConfig struct { ImageRepoSecret string `yaml:"imageRepoSecret" json:"imageRepoSecret"` } +type MetricsConfig struct { + Port uint16 `yaml:"port" json:"port" default:"49100"` +} + type TapConfig struct { Docker DockerConfig `yaml:"docker" json:"docker"` Proxy ProxyConfig `yaml:"proxy" json:"proxy"` @@ -154,6 +158,7 @@ type TapConfig struct { ReplayDisabled bool `yaml:"replayDisabled" json:"replayDisabled" default:"false"` Capabilities CapabilitiesConfig `yaml:"capabilities" json:"capabilities"` GlobalFilter string `yaml:"globalFilter" json:"globalFilter"` + Metrics MetricsConfig `yaml:"metrics" json:"metrics"` } func (config *TapConfig) PodRegex() *regexp.Regexp { diff --git a/helm-chart/README.md b/helm-chart/README.md index dd24c58a2..4b86d8327 100644 --- a/helm-chart/README.md +++ b/helm-chart/README.md @@ -104,6 +104,10 @@ helm install kubeshark kubeshark/kubeshark \ --set tap.ipv6=false ``` +## Metrics + +Please refer to [metrics](./metrics.md) documentation for details. + ## Configuration | Parameter | Description | Default | @@ -168,6 +172,7 @@ helm install kubeshark kubeshark/kubeshark \ | `scripting.env` | Environment variables for the scripting | `{}` | | `scripting.source` | Source directory of the scripts | `""` | | `scripting.watchScripts` | Enable watch mode for the scripts in source directory | `true` | +| `tap.metrics.port` | Pod port used to expose Prometheus metrics | `49100` | KernelMapping pairs kernel versions with a DriverContainer image. Kernel versions can be matched diff --git a/helm-chart/metrics.md b/helm-chart/metrics.md new file mode 100644 index 000000000..bd1ca4571 --- /dev/null +++ b/helm-chart/metrics.md @@ -0,0 +1,51 @@ +# Metrics + +Kubeshark provides metrics from `worker` components. +It can be useful for monitoring and debugging purpose. + +## Configuration + +By default, Kubeshark uses port `49100` to expose metrics via service `kubeshark-worker-metrics`. + +In case you use [kube-prometheus-stack] (https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) community Helm chart, additional scrape configuration for Kubeshark worker metrics endpoint can be configured with values: + +``` +prometheus: + enabled: true + prometheusSpec: + additionalScrapeConfigs: | + - job_name: 'kubeshark-worker-metrics' + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_pod_node_name] + target_label: node + - source_labels: [__meta_kubernetes_endpoint_port_name] + action: keep + regex: ^metrics$ + - source_labels: [__address__, __meta_kubernetes_endpoint_port_number] + action: replace + regex: ([^:]+)(?::\d+)? + replacement: $1:49100 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) +``` + + +## Available metrics + +| Name | Type | Description | +| --- | --- | --- | +| kubeshark_received_packets_total | Counter | Total number of packets received | +| kubeshark_dropped_packets_total | Counter | Total number of packets dropped | +| kubeshark_processed_bytes_total | Counter | Total number of bytes processed | +| kubeshark_tcp_packets_total | Counter | Total number of TCP packets | +| kubeshark_dns_packets_total | Counter | Total number of DNS packets | +| kubeshark_icmp_packets_total | Counter | Total number of ICMP packets | +| kubeshark_reassembled_tcp_payloads_total | Counter | Total number of reassembled TCP payloads | +| kubeshark_matched_pairs_total | Counter | Total number of matched pairs | +| kubeshark_dropped_tcp_streams_total | Counter | Total number of dropped TCP streams | +| kubeshark_live_tcp_streams | Gauge | Number of live TCP streams | \ No newline at end of file diff --git a/helm-chart/templates/09-worker-daemon-set.yaml b/helm-chart/templates/09-worker-daemon-set.yaml index 5e0f152c5..51b5bc598 100644 --- a/helm-chart/templates/09-worker-daemon-set.yaml +++ b/helm-chart/templates/09-worker-daemon-set.yaml @@ -49,6 +49,8 @@ spec: - any - -port - '{{ .Values.tap.proxy.worker.srvPort }}' + - -metrics-port + - '{{ .Values.tap.metrics.port }}' {{- if .Values.tap.serviceMesh }} - -servicemesh {{- end }} @@ -63,6 +65,10 @@ spec: image: '{{ .Values.tap.docker.registry }}/worker:{{ not (eq .Values.tap.docker.tag "") | ternary .Values.tap.docker.tag (printf "v%s" .Chart.Version) }}' imagePullPolicy: {{ .Values.tap.docker.imagePullPolicy }} name: sniffer + ports: + - containerPort: {{ .Values.tap.metrics.port }} + protocol: TCP + name: metrics env: - name: POD_NAME valueFrom: diff --git a/helm-chart/templates/16-worker-service-metrics.yaml b/helm-chart/templates/16-worker-service-metrics.yaml new file mode 100644 index 000000000..bbe3cf648 --- /dev/null +++ b/helm-chart/templates/16-worker-service-metrics.yaml @@ -0,0 +1,18 @@ +--- +kind: Service +apiVersion: v1 +metadata: + name: kubeshark-worker-metrics + namespace: {{ .Release.Namespace }} + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '{{ .Values.tap.metrics.port }}' +spec: + selector: + app.kubeshark.co/app: worker + {{- include "kubeshark.labels" . | nindent 4 }} + ports: + - name: metrics + protocol: TCP + port: {{ .Values.tap.metrics.port }} + targetPort: {{ .Values.tap.metrics.port }} diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 7cd38a2a6..ede26e329 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -91,6 +91,8 @@ tap: - SYS_RESOURCE - CHECKPOINT_RESTORE globalFilter: "" + metrics: + port: 49100 logs: file: "" kube: