diff --git a/tests/metrics/disk/cassandra_kubernetes/cassandra.sh b/tests/metrics/disk/cassandra_kubernetes/cassandra.sh new file mode 100755 index 0000000000..b6c636548a --- /dev/null +++ b/tests/metrics/disk/cassandra_kubernetes/cassandra.sh @@ -0,0 +1,198 @@ +#!/bin/bash +# +# Copyright (c) 2022-2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o pipefail + +SCRIPT_PATH=$(dirname "$(readlink -f "$0")") + +source "${SCRIPT_PATH}/../../lib/common.bash" +TEST_NAME="${TEST_NAME:-cassandra}" +cassandra_file=$(mktemp cassandraresults.XXXXXXXXXX) +cassandra_read_file=$(mktemp cassandrareadresults.XXXXXXXXXX) + +function remove_tmp_file() { + rm -rf "${cassandra_file}" "${cassandra_read_file}" +} + +trap remove_tmp_file EXIT + +function cassandra_write_test() { + cassandra_start + export pod_name="cassandra-0" + export write_cmd="/usr/local/apache-cassandra-3.11.2/tools/bin/cassandra-stress write n=1000000 cl=one -mode native cql3 -schema keyspace="keyspace1" -pop seq=1..1000000 -node cassandra" + number_of_retries="50" + for _ in $(seq 1 "$number_of_retries"); do + if kubectl exec -i cassandra-0 -- sh -c 'nodetool status' | grep Up; then + ok="1" + break; + fi + sleep 1 + done + # This is needed to wait that cassandra is up + sleep 30 + kubectl exec -i cassandra-0 -- sh -c "$write_cmd" > "${cassandra_file}" + write_op_rate=$(cat "${cassandra_file}" | grep -e "Op rate" | cut -d':' -f2 | sed -e 's/^[ \t]*//' | cut -d ' ' -f1) + write_latency_mean=$(cat "${cassandra_file}" | grep -e "Latency mean" | cut -d':' -f2 | sed -e 's/^[ \t]*//' | cut -d ' ' -f1) + write_latency_95th=$(cat "${cassandra_file}" | grep -e "Latency 95th percentile" | cut -d':' -f2 | sed -e 's/^[ \t]*//' | cut -d ' ' -f1) + write_latency_99th=$(cat "${cassandra_file}" | grep -e "Latency 99th percentile" | cut -d':' -f2 | sed -e 's/^[ \t]*//' | cut -d ' ' -f1) + write_latency_median=$(cat "${cassandra_file}" | grep -e "Latency median" | cut -d':' -f2 | sed -e 's/^[ \t]*//' | cut -d ' ' -f1) + + export read_cmd="/usr/local/apache-cassandra-3.11.2/tools/bin/cassandra-stress read n=200000 -rate threads=50" + kubectl exec -i cassandra-0 -- sh -c "$read_cmd" > "${cassandra_read_file}" + read_op_rate=$(cat "${cassandra_read_file}" | grep -e "Op rate" | cut -d':' -f2 | sed -e 's/^[ \t]*//' | cut -d ' ' -f1) + read_latency_mean=$(cat "${cassandra_read_file}" | grep -e "Latency mean" | cut -d':' -f2 | sed -e 's/^[ \t]*//' | cut -d ' ' -f1) + read_latency_95th=$(cat "${cassandra_read_file}" | grep -e "Latency 95th percentile" | cut -d':' -f2 | sed -e 's/^[ \t]*//' | cut -d ' ' -f1) + read_latency_99th=$(cat "${cassandra_read_file}" | grep -e "Latency 99th percentile" | cut -d':' -f2 | sed -e 's/^[ \t]*//' | cut -d ' ' -f1) + read_latency_median=$(cat "${cassandra_read_file}" | grep -e "Latency median" | cut -d':' -f2 | sed -e 's/^[ \t]*//' | cut -d ' ' -f1) + + metrics_json_init + # Save configuration + metrics_json_start_array + + local json="$(cat << EOF + { + "Write Op rate": { + "Result" : "$write_op_rate", + "Units" : "op/s" + }, + "Write Latency Mean": { + "Result" : "$write_latency_mean", + "Units" : "ms" + }, + "Write Latency 95th percentile": { + "Result" : "$write_latency_95th", + "Units" : "ms" + }, + "Write Latency 99th percentile": { + "Result" : "$write_latency_99th", + "Units" : "ms" + }, + "Write Latency Median" : { + "Result" : "$write_latency_median", + "Units" : "ms" + }, + "Read Op rate": { + "Result" : "$read_op_rate", + "Units" : "op/s" + }, + "Read Latency Mean": { + "Result" : "$read_latency_mean", + "Units" : "ms" + }, + "Read Latency 95th percentile": { + "Result" : "$read_latency_95th", + "Units" : "ms" + }, + "Read Latency 99th percentile": { + "Result" : "$read_latency_99th", + "Units" : "ms" + }, + "Read Latency Median" : { + "Result" : "$read_latency_median", + "Units" : "ms" + } + } +EOF +)" + + metrics_json_add_array_element "$json" + metrics_json_end_array "Results" + + metrics_json_save + cassandra_cleanup +} + +function cassandra_start() { + cmds=("bc" "jq") + check_cmds "${cmds[@]}" + + # Check no processes are left behind + check_processes + + export service_name="cassandra" + export app_name="cassandra" + + wait_time=20 + sleep_time=2 + + vol_capacity="3Gi" + volume_name="block-loop-pv" + volume_claim="block-loop-pvc" + + # Create Loop Device + export tmp_disk_image=$(mktemp --tmpdir disk.XXXXXX.img) + truncate "$tmp_disk_image" --size "3GB" + export loop_dev=$(sudo losetup -f) + sudo losetup "$loop_dev" "$tmp_disk_image" + + # Create Storage Class + kubectl create -f "${SCRIPT_PATH}/volume/block-local-storage.yaml" + + # Create Persistent Volume + export tmp_pv_yaml=$(mktemp --tmpdir block_persistent_vol.XXXXX.yaml) + sed -e "s|LOOP_DEVICE|${loop_dev}|" "${SCRIPT_PATH}/volume/block-loop-pv.yaml" > "$tmp_pv_yaml" + sed -i "s|HOSTNAME|$(hostname | awk '{print tolower($0)}')|" "$tmp_pv_yaml" + sed -i "s|CAPACITY|${vol_capacity}|" "$tmp_pv_yaml" + + kubectl create -f "$tmp_pv_yaml" + cmd="kubectl get pv/${volume_name} | grep Available" + waitForProcess "$wait_time" "$sleep_time" "$cmd" + + # Create Persistent Volume Claim + export tmp_pvc_yaml=$(mktemp --tmpdir block_persistent_vol.XXXXX.yaml) + sed -e "s|CAPACITY|${vol_capacity}|" "${SCRIPT_PATH}/volume/block-loop-pvc.yaml" > "$tmp_pvc_yaml" + kubectl create -f "$tmp_pvc_yaml" + + # Create service + kubectl create -f "${SCRIPT_PATH}/runtimeclass_workloads/cassandra-service.yaml" + + # Check service + kubectl get svc | grep "$service_name" + + # Create workload using volume + ctr_dev_path="/dev/xda" + export tmp_pod_yaml=$(mktemp --tmpdir pod-pv.XXXXX.yaml) + sed -e "s|DEVICE_PATH|${ctr_dev_path}|" "${SCRIPT_PATH}/runtimeclass_workloads/cassandra-statefulset.yaml" > "$tmp_pod_yaml" + kubectl create -f "$tmp_pod_yaml" + cmd="kubectl rollout status --watch --timeout=120s statefulset/$app_name" + waitForProcess "$wait_time" "$sleep_time" "$cmd" + + # Verify persistent volume claim is bound + kubectl get pvc | grep "Bound" + + # Check pods are running + cmd="kubectl get pods -o jsonpath='{.items[*].status.phase}' | grep Running" + waitForProcess "$wait_time" "$sleep_time" "$cmd" +} + +function cassandra_cleanup() { + kubectl patch pvc block-loop-pvc -p '{"metadata":{"finalizers":null}}' + kubectl delete pvc block-loop-pvc --force + kubectl patch pv block-loop-pv -p '{"metadata":{"finalizers":null}}' + kubectl delete pv block-loop-pv --force + kubectl delete svc "$service_name" + kubectl delete pod -l app="$app_name" + kubectl delete storageclass block-local-storage + kubectl delete statefulsets "$app_name" + + # Delete temporary yaml files + rm -f "$tmp_pv_yaml" + rm -f "$tmp_pvc_yaml" + rm -f "$tmp_pod_yaml" + + # Remove image and loop device + sudo losetup -d "$loop_dev" + rm -f "$tmp_disk_image" + + check_processes +} + +function main() { + init_env + cassandra_write_test +} + +main "$@" diff --git a/tests/metrics/disk/cassandra_kubernetes/runtimeclass_workloads/cassandra-service.yaml b/tests/metrics/disk/cassandra_kubernetes/runtimeclass_workloads/cassandra-service.yaml new file mode 100644 index 0000000000..26e43d09cc --- /dev/null +++ b/tests/metrics/disk/cassandra_kubernetes/runtimeclass_workloads/cassandra-service.yaml @@ -0,0 +1,17 @@ +# +# Copyright (c) 2022-2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: Service +metadata: + labels: + app: cassandra + name: cassandra +spec: + clusterIP: None + ports: + - port: 9042 + selector: + app: cassandra diff --git a/tests/metrics/disk/cassandra_kubernetes/runtimeclass_workloads/cassandra-statefulset.yaml b/tests/metrics/disk/cassandra_kubernetes/runtimeclass_workloads/cassandra-statefulset.yaml new file mode 100644 index 0000000000..fdc10df510 --- /dev/null +++ b/tests/metrics/disk/cassandra_kubernetes/runtimeclass_workloads/cassandra-statefulset.yaml @@ -0,0 +1,56 @@ +# +# Copyright (c) 2022-2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: "apps/v1" +kind: StatefulSet +metadata: + name: cassandra +spec: + serviceName: cassandra + replicas: 1 + selector: + matchLabels: + app: cassandra + template: + metadata: + labels: + app: cassandra + spec: + runtimeClassName: kata + containers: + - name: cassandra + image: gcr.io/google-samples/cassandra:v13 + imagePullPolicy: Always + ports: + - containerPort: 7000 + name: intra-node + - containerPort: 7001 + name: tls-intra-node + - containerPort: 7199 + name: jmx + - containerPort: 9042 + name: cql + env: + - name: CASSANDRA_SEEDS + value: cassandra-0.cassandra.default.svc.cluster.local + - name: MAX_HEAP_SIZE + value: 256M + - name: HEAP_NEWSIZE + value: 100M + - name: CASSANDRA_CLUSTER_NAME + value: "Cassandra" + - name: CASSANDRA_DC + value: "DC1" + - name: CASSANDRA_RACK + value: "Rack1" + - name: CASSANDRA_ENDPOINT_SNITCH + value: GossipingPropertyFileSnitch + volumeDevices: + - devicePath: DEVICE_PATH + name: my-volume + volumes: + - name: my-volume + persistentVolumeClaim: + claimName: block-loop-pvc diff --git a/tests/metrics/disk/cassandra_kubernetes/volume/block-local-storage.yaml b/tests/metrics/disk/cassandra_kubernetes/volume/block-local-storage.yaml new file mode 100644 index 0000000000..5f02094088 --- /dev/null +++ b/tests/metrics/disk/cassandra_kubernetes/volume/block-local-storage.yaml @@ -0,0 +1,6 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: block-local-storage +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer diff --git a/tests/metrics/disk/cassandra_kubernetes/volume/block-loop-pv.yaml b/tests/metrics/disk/cassandra_kubernetes/volume/block-loop-pv.yaml new file mode 100644 index 0000000000..b657a059cf --- /dev/null +++ b/tests/metrics/disk/cassandra_kubernetes/volume/block-loop-pv.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: block-loop-pv +spec: + capacity: + storage: CAPACITY + volumeMode: Block + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: block-local-storage + local: + path: LOOP_DEVICE + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - HOSTNAME diff --git a/tests/metrics/disk/cassandra_kubernetes/volume/block-loop-pvc.yaml b/tests/metrics/disk/cassandra_kubernetes/volume/block-loop-pvc.yaml new file mode 100644 index 0000000000..e796af206a --- /dev/null +++ b/tests/metrics/disk/cassandra_kubernetes/volume/block-loop-pvc.yaml @@ -0,0 +1,12 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: block-loop-pvc +spec: + accessModes: + - ReadWriteOnce + storageClassName: block-local-storage + volumeMode: Block + resources: + requests: + storage: CAPACITY