Merge pull request #7626 from GabyCT/topic/cassandrak

metrics: Add Cassandra Kubernetes benchmark for kata metrics

tests/metrics/disk/cassandra_kubernetes/cassandra.sh

@@ -0,0 +1,198 @@
+#!/bin/bash
+#
+# Copyright (c) 2022-2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -o pipefail
+
+SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
+
+source "${SCRIPT_PATH}/../../lib/common.bash"
+TEST_NAME="${TEST_NAME:-cassandra}"
+cassandra_file=$(mktemp cassandraresults.XXXXXXXXXX)
+cassandra_read_file=$(mktemp cassandrareadresults.XXXXXXXXXX)
+
+function remove_tmp_file() {
+	rm -rf "${cassandra_file}" "${cassandra_read_file}"
+}
+
+trap remove_tmp_file EXIT
+
+function cassandra_write_test() {
+	cassandra_start
+	export pod_name="cassandra-0"
+	export write_cmd="/usr/local/apache-cassandra-3.11.2/tools/bin/cassandra-stress write n=1000000 cl=one -mode native cql3 -schema keyspace="keyspace1" -pop seq=1..1000000 -node cassandra"
+ 	number_of_retries="50"
+	for _ in $(seq 1 "$number_of_retries"); do
+		if kubectl exec -i cassandra-0 -- sh -c 'nodetool status' | grep Up; then
+			ok="1"
+ 			break;
+		fi
+ 		sleep 1
+	done
+	# This is needed to wait that cassandra is up
+	sleep 30
+	kubectl exec -i cassandra-0 -- sh -c "$write_cmd" > "${cassandra_file}"
+	write_op_rate=$(cat "${cassandra_file}" | grep -e "Op rate" | cut -d':' -f2  | sed -e 's/^[ \t]*//' | cut -d ' ' -f1)
+	write_latency_mean=$(cat "${cassandra_file}" | grep -e "Latency mean" | cut -d':' -f2  | sed -e 's/^[ \t]*//' | cut -d ' ' -f1)
+	write_latency_95th=$(cat "${cassandra_file}" | grep -e "Latency 95th percentile" | cut -d':' -f2  | sed -e 's/^[ \t]*//' | cut -d ' ' -f1)
+	write_latency_99th=$(cat "${cassandra_file}" | grep -e "Latency 99th percentile" | cut -d':' -f2  | sed -e 's/^[ \t]*//' | cut -d ' ' -f1)
+	write_latency_median=$(cat "${cassandra_file}" | grep -e "Latency median" | cut -d':' -f2  | sed -e 's/^[ \t]*//' | cut -d ' ' -f1)
+
+	export read_cmd="/usr/local/apache-cassandra-3.11.2/tools/bin/cassandra-stress read n=200000 -rate threads=50"
+	kubectl exec -i cassandra-0 -- sh -c "$read_cmd" > "${cassandra_read_file}"
+	read_op_rate=$(cat "${cassandra_read_file}" | grep -e "Op rate" | cut -d':' -f2  | sed -e 's/^[ \t]*//' | cut -d ' ' -f1)
+	read_latency_mean=$(cat "${cassandra_read_file}" | grep -e "Latency mean" | cut -d':' -f2  | sed -e 's/^[ \t]*//' | cut -d ' ' -f1)
+	read_latency_95th=$(cat "${cassandra_read_file}" | grep -e "Latency 95th percentile" | cut -d':' -f2  | sed -e 's/^[ \t]*//' | cut -d ' ' -f1)
+	read_latency_99th=$(cat "${cassandra_read_file}" | grep -e "Latency 99th percentile" | cut -d':' -f2  | sed -e 's/^[ \t]*//' | cut -d ' ' -f1)
+	read_latency_median=$(cat "${cassandra_read_file}" | grep -e "Latency median" | cut -d':' -f2  | sed -e 's/^[ \t]*//' | cut -d ' ' -f1)
+
+	metrics_json_init
+	# Save configuration
+	metrics_json_start_array
+
+	local json="$(cat << EOF
+	{
+		"Write Op rate": {
+			"Result" : "$write_op_rate",
+			"Units" : "op/s"
+		},
+		"Write Latency Mean": {
+			"Result" : "$write_latency_mean",
+			"Units" : "ms"
+		},
+		"Write Latency 95th percentile": {
+			"Result" : "$write_latency_95th",
+			"Units" : "ms"
+		},
+		"Write Latency 99th percentile": {
+			"Result" : "$write_latency_99th",
+			"Units" : "ms"
+		},
+		"Write Latency Median" : {
+			"Result" : "$write_latency_median",
+			"Units" : "ms"
+		},
+		"Read Op rate": {
+			"Result" : "$read_op_rate",
+			"Units" : "op/s"
+		},
+		"Read Latency Mean": {
+			"Result" : "$read_latency_mean",
+			"Units" : "ms"
+		},
+		"Read Latency 95th percentile": {
+			"Result" : "$read_latency_95th",
+			"Units" : "ms"
+		},
+		"Read Latency 99th percentile": {
+			"Result" : "$read_latency_99th",
+			"Units" : "ms"
+		},
+		"Read Latency Median" : {
+			"Result" : "$read_latency_median",
+			"Units" : "ms"
+		}
+	}
+EOF
+)"
+
+	metrics_json_add_array_element "$json"
+	metrics_json_end_array "Results"
+
+	metrics_json_save
+	cassandra_cleanup
+}
+
+function cassandra_start() {
+	cmds=("bc" "jq")
+	check_cmds "${cmds[@]}"
+
+	# Check no processes are left behind
+	check_processes
+
+	export service_name="cassandra"
+	export app_name="cassandra"
+
+	wait_time=20
+ 	sleep_time=2
+
+	vol_capacity="3Gi"
+	volume_name="block-loop-pv"
+	volume_claim="block-loop-pvc"
+
+	# Create Loop Device
+	export tmp_disk_image=$(mktemp --tmpdir disk.XXXXXX.img)
+	truncate "$tmp_disk_image" --size "3GB"
+	export loop_dev=$(sudo losetup -f)
+	sudo losetup "$loop_dev" "$tmp_disk_image"
+
+	# Create Storage Class
+	kubectl create -f "${SCRIPT_PATH}/volume/block-local-storage.yaml"
+
+	# Create Persistent Volume
+	export tmp_pv_yaml=$(mktemp --tmpdir block_persistent_vol.XXXXX.yaml)
+	sed -e "s|LOOP_DEVICE|${loop_dev}|" "${SCRIPT_PATH}/volume/block-loop-pv.yaml" > "$tmp_pv_yaml"
+	sed -i "s|HOSTNAME|$(hostname | awk '{print tolower($0)}')|" "$tmp_pv_yaml"
+	sed -i "s|CAPACITY|${vol_capacity}|" "$tmp_pv_yaml"
+
+	kubectl create -f "$tmp_pv_yaml"
+	cmd="kubectl get pv/${volume_name} | grep Available"
+	waitForProcess "$wait_time" "$sleep_time" "$cmd"
+
+	# Create Persistent Volume Claim
+	export tmp_pvc_yaml=$(mktemp --tmpdir block_persistent_vol.XXXXX.yaml)
+	sed -e "s|CAPACITY|${vol_capacity}|" "${SCRIPT_PATH}/volume/block-loop-pvc.yaml" > "$tmp_pvc_yaml"
+	kubectl create -f "$tmp_pvc_yaml"
+
+	# Create service
+	kubectl create -f "${SCRIPT_PATH}/runtimeclass_workloads/cassandra-service.yaml"
+
+	# Check service
+	kubectl get svc | grep "$service_name"
+
+	# Create workload using volume
+	ctr_dev_path="/dev/xda"
+	export tmp_pod_yaml=$(mktemp --tmpdir pod-pv.XXXXX.yaml)
+	sed -e "s|DEVICE_PATH|${ctr_dev_path}|" "${SCRIPT_PATH}/runtimeclass_workloads/cassandra-statefulset.yaml" > "$tmp_pod_yaml"
+	kubectl create -f "$tmp_pod_yaml"
+	cmd="kubectl rollout status --watch --timeout=120s statefulset/$app_name"
+ 	waitForProcess "$wait_time" "$sleep_time" "$cmd"
+
+	# Verify persistent volume claim is bound
+	kubectl get pvc | grep "Bound"
+
+	# Check pods are running
+	cmd="kubectl get pods -o jsonpath='{.items[*].status.phase}' | grep Running"
+	waitForProcess "$wait_time" "$sleep_time" "$cmd"
+}
+
+function cassandra_cleanup() {
+	kubectl patch pvc block-loop-pvc -p '{"metadata":{"finalizers":null}}'
+	kubectl delete pvc block-loop-pvc --force
+	kubectl patch pv block-loop-pv -p '{"metadata":{"finalizers":null}}'
+	kubectl delete pv block-loop-pv --force
+	kubectl delete svc "$service_name"
+	kubectl delete pod -l app="$app_name"
+	kubectl delete storageclass block-local-storage
+	kubectl delete statefulsets "$app_name"
+
+	# Delete temporary yaml files
+	rm -f "$tmp_pv_yaml"
+	rm -f "$tmp_pvc_yaml"
+	rm -f "$tmp_pod_yaml"
+
+	# Remove image and loop device
+	sudo losetup -d "$loop_dev"
+	rm -f "$tmp_disk_image"
+
+	check_processes
+}
+
+function main() {
+	init_env
+	cassandra_write_test
+}
+
+main "$@"

tests/metrics/disk/cassandra_kubernetes/runtimeclass_workloads/cassandra-service.yaml

@@ -0,0 +1,17 @@
+#
+# Copyright (c) 2022-2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: cassandra
+  name: cassandra
+spec:
+  clusterIP: None
+  ports:
+  - port: 9042
+  selector:
+    app: cassandra

tests/metrics/disk/cassandra_kubernetes/runtimeclass_workloads/cassandra-statefulset.yaml

@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2022-2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+apiVersion: "apps/v1"
+kind: StatefulSet
+metadata:
+  name: cassandra
+spec:
+  serviceName: cassandra
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cassandra
+  template:
+    metadata:
+      labels:
+        app: cassandra
+    spec:
+      runtimeClassName: kata
+      containers:
+        - name: cassandra
+          image: gcr.io/google-samples/cassandra:v13
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 7000
+              name: intra-node
+            - containerPort: 7001
+              name: tls-intra-node
+            - containerPort: 7199
+              name: jmx
+            - containerPort: 9042
+              name: cql
+          env:
+            - name: CASSANDRA_SEEDS
+              value: cassandra-0.cassandra.default.svc.cluster.local
+            - name: MAX_HEAP_SIZE
+              value: 256M
+            - name: HEAP_NEWSIZE
+              value: 100M
+            - name: CASSANDRA_CLUSTER_NAME
+              value: "Cassandra"
+            - name: CASSANDRA_DC
+              value: "DC1"
+            - name: CASSANDRA_RACK
+              value: "Rack1"
+            - name: CASSANDRA_ENDPOINT_SNITCH
+              value: GossipingPropertyFileSnitch
+          volumeDevices:
+            - devicePath: DEVICE_PATH
+              name: my-volume
+      volumes:
+        - name: my-volume
+          persistentVolumeClaim:
+            claimName: block-loop-pvc

tests/metrics/disk/cassandra_kubernetes/volume/block-local-storage.yaml

@@ -0,0 +1,6 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: block-local-storage
+provisioner: kubernetes.io/no-provisioner
+volumeBindingMode: WaitForFirstConsumer

tests/metrics/disk/cassandra_kubernetes/volume/block-loop-pv.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: block-loop-pv
+spec:
+  capacity:
+    storage: CAPACITY
+  volumeMode: Block
+  accessModes:
+  - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: block-local-storage
+  local:
+    path: LOOP_DEVICE
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+          - key: kubernetes.io/hostname
+            operator: In
+            values:
+            - HOSTNAME

tests/metrics/disk/cassandra_kubernetes/volume/block-loop-pvc.yaml

@@ -0,0 +1,12 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: block-loop-pvc
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: block-local-storage
+  volumeMode: Block
+  resources:
+    requests:
+      storage: CAPACITY