diff --git a/src/tools/csi-kata-directvolume/deploy/deploy.sh b/src/tools/csi-kata-directvolume/deploy/deploy.sh new file mode 100644 index 0000000000..0cde476efa --- /dev/null +++ b/src/tools/csi-kata-directvolume/deploy/deploy.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2023 Ant Group +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e +set -o pipefail + +BASE_DIR=$(dirname "$0") + +${BASE_DIR}/rbac-deploy.sh +${BASE_DIR}/directvol-deploy.sh diff --git a/src/tools/csi-kata-directvolume/deploy/directvol-deploy.sh b/src/tools/csi-kata-directvolume/deploy/directvol-deploy.sh new file mode 100755 index 0000000000..43dd54e492 --- /dev/null +++ b/src/tools/csi-kata-directvolume/deploy/directvol-deploy.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# +# Copyright 2017 The Kubernetes Authors. +# Copyright (c) 2023 Ant Group +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e +set -o pipefail + +BASE_DIR=$(dirname "$0") + +# KUBELET_DATA_DIR can be set to replace the default /var/lib/kubelet. +# All nodes must use the same directory. +default_kubelet_data_dir=/var/lib/kubelet +: ${KUBELET_DATA_DIR:=${default_kubelet_data_dir}} + +# namespace kata-directvolume +DIRECTVOL_NAMESPACE="kata-directvolume" + +# Some images are not affected by *_REGISTRY/*_TAG and IMAGE_* variables. +# The default is to update unless explicitly excluded. +update_image () { + case "$1" in socat) return 1;; esac +} + +run () { + echo "$@" >&2 + "$@" +} + +# deploy kata directvolume plugin and registrar sidecar +echo "deploying kata directvolume components" +for i in $(ls ${BASE_DIR}/kata-directvolume/csi-directvol-*.yaml | sort); do + echo " $i" + modified="$(cat "$i" | sed -e "s;${default_kubelet_data_dir}/;${KUBELET_DATA_DIR}/;" | while IFS= read -r line; do + nocomments="$(echo "$line" | sed -e 's/ *#.*$//')" + if echo "$nocomments" | grep -q '^[[:space:]]*image:[[:space:]]*'; then + # Split 'image: quay.io/k8scsi/csi-attacher:vx.y.z' + # into image (quay.io/k8scsi/csi-attacher:vx.y.z), + # registry (quay.io/k8scsi), + # name (csi-attacher), + # tag (vx.y.z). + image=$(echo "$nocomments" | sed -e 's;.*image:[[:space:]]*;;') + registry=$(echo "$image" | sed -e 's;\(.*\)/.*;\1;') + name=$(echo "$image" | sed -e 's;.*/\([^:]*\).*;\1;') + tag=$(echo "$image" | sed -e 's;.*:;;') + + # Variables are with underscores and upper case. + varname=$(echo $name | tr - _ | tr a-z A-Z) + + # Now replace registry and/or tag, if set as env variables. + # If not set, the replacement is the same as the original value. + # Only do this for the images which are meant to be configurable. + if update_image "$name"; then + prefix=$(eval echo \${${varname}_REGISTRY:-${IMAGE_REGISTRY:-${registry}}}/ | sed -e 's;none/;;') + if [ "$IMAGE_TAG" = "canary" ] && + [ -f ${BASE_DIR}/canary-blacklist.txt ] && + grep -q "^$name\$" ${BASE_DIR}/canary-blacklist.txt; then + # Ignore IMAGE_TAG=canary for this particular image because its + # canary image is blacklisted in the deployment blacklist. + suffix=$(eval echo :\${${varname}_TAG:-${tag}}) + else + suffix=$(eval echo :\${${varname}_TAG:-${IMAGE_TAG:-${tag}}}) + fi + line="$(echo "$nocomments" | sed -e "s;$image;${prefix}${name}${suffix};")" + fi + echo "kata-directvolume plugin using $line" >&2 + fi + if ! $have_csistoragecapacity; then + line="$(echo "$line" | grep -v -e 'storageCapacity: true' -e '--enable-capacity')" + fi + echo "$line" + done)" + if ! echo "$modified" | kubectl apply -f -; then + echo "modified version of $i:" + echo "$modified" + exit 1 + fi +done + +wait_for_daemonset () { + retries=10 + while [ $retries -ge 0 ]; do + ready=$(kubectl get -n $1 daemonset $2 -o jsonpath="{.status.numberReady}") + required=$(kubectl get -n $1 daemonset $2 -o jsonpath="{.status.desiredNumberScheduled}") + if [ $ready -gt 0 ] && [ $ready -eq $required ]; then + return 0 + fi + retries=$((retries - 1)) + sleep 3 + done + return 1 +} + + +# Wait until the DaemonSet is running on all nodes. +if ! wait_for_daemonset ${DIRECTVOL_NAMESPACE} csi-kata-directvol-plugin; then + echo + echo "driver not ready" + echo "Deployment:" + (set +e; set -x; kubectl describe all,role,clusterrole,rolebinding,clusterrolebinding,serviceaccount,storageclass,csidriver --all-namespaces -l app.kubernetes.io/instance=directvolume.csi.katacontainers.io) + echo + echo "Pod logs:" + kubectl get pods -l app.kubernetes.io/instance=directvolume.csi.katacontainers.io --all-namespaces -o=jsonpath='{range .items[*]}{.metadata.name}{" "}{range .spec.containers[*]}{.name}{" "}{end}{"\n"}{end}' | while read -r pod containers; do + for c in $containers; do + echo + (set +e; set -x; kubectl logs $pod $c) + done + done + exit 1 +fi + +kubectl get po,ds -A diff --git a/src/tools/csi-kata-directvolume/deploy/directvol-destroy.sh b/src/tools/csi-kata-directvolume/deploy/directvol-destroy.sh new file mode 100755 index 0000000000..59fca37f6b --- /dev/null +++ b/src/tools/csi-kata-directvolume/deploy/directvol-destroy.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# +# Copyright 2017 The Kubernetes Authors. +# Copyright (c) 2023 Ant Group +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e +set -o pipefail + +# Deleting all the resources installed by the directvol-deploy script. +# Every resource in the driver installation has the label representing the installation instance. +# Using app.kubernetes.io/instance: directvolume.csi.katacontainers.io and app.kubernetes.io/part-of: +# csi-driver-kata-directvolume labels to identify the installation set +kubectl delete all --all-namespaces -l app.kubernetes.io/instance=directvolume.csi.katacontainers.io,app.kubernetes.io/part-of=csi-driver-kata-directvolume --wait=true +kubectl delete role,clusterrole,rolebinding,clusterrolebinding,serviceaccount,storageclass,csidriver --all-namespaces -l app.kubernetes.io/instance=directvolume.csi.katacontainers.io,app.kubernetes.io/part-of=csi-driver-kata-directvolume --wait=true diff --git a/src/tools/csi-kata-directvolume/deploy/kata-directvolume/csi-directvol-driverinfo.yaml b/src/tools/csi-kata-directvolume/deploy/kata-directvolume/csi-directvol-driverinfo.yaml new file mode 100644 index 0000000000..f12b2d5dd1 --- /dev/null +++ b/src/tools/csi-kata-directvolume/deploy/kata-directvolume/csi-directvol-driverinfo.yaml @@ -0,0 +1,21 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: directvolume.csi.katacontainers.io + labels: + app.kubernetes.io/instance: directvolume.csi.katacontainers.io + app.kubernetes.io/part-of: csi-driver-kata-directvolume + app.kubernetes.io/name: directvolume.csi.katacontainers.io + app.kubernetes.io/component: csi-driver +spec: + # Supports persistent volume. + volumeLifecycleModes: + - Persistent + # To determine at runtime which mode a volume uses, pod info. + podInfoOnMount: true + # No attacher needed. + attachRequired: false + storageCapacity: false + # Kubernetes may use fsGroup to change permissions and ownership + # of the volume to match user requested fsGroup in the pod's SecurityPolicy + fsGroupPolicy: File diff --git a/src/tools/csi-kata-directvolume/deploy/kata-directvolume/csi-directvol-plugin.yaml b/src/tools/csi-kata-directvolume/deploy/kata-directvolume/csi-directvol-plugin.yaml new file mode 100644 index 0000000000..a243956254 --- /dev/null +++ b/src/tools/csi-kata-directvolume/deploy/kata-directvolume/csi-directvol-plugin.yaml @@ -0,0 +1,190 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + namespace: kata-directvolume + name: csi-kata-directvol-plugin + labels: + app.kubernetes.io/instance: directvolume.csi.katacontainers.io + app.kubernetes.io/part-of: csi-driver-kata-directvolume + app.kubernetes.io/name: csi-kata-directvol-plugin + app.kubernetes.io/component: plugin +spec: + selector: + matchLabels: + app.kubernetes.io/instance: directvolume.csi.katacontainers.io + app.kubernetes.io/part-of: csi-driver-kata-directvolume + app.kubernetes.io/name: csi-kata-directvol-plugin + app.kubernetes.io/component: plugin + template: + metadata: + labels: + app.kubernetes.io/instance: directvolume.csi.katacontainers.io + app.kubernetes.io/part-of: csi-driver-kata-directvolume + app.kubernetes.io/name: csi-kata-directvol-plugin + app.kubernetes.io/component: plugin + spec: + serviceAccountName: csi-provisioner + containers: + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0 + args: + - -v=3 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --worker-threads=5 + #- --enable-capacity + #- --capacity-ownerref-level=0 # pod is owner + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + + - name: node-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0 + args: + - --v=3 + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-kata-directvolume/csi.sock + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /registration + name: registration-dir + - mountPath: /csi-persist-data + name: csi-persist-data + + - name: kata-directvolume + # build and push it into registry + image: localhost/kata-directvolume:v1.0.18 + args: + - --drivername=directvolume.csi.katacontainers.io + - --v=5 + - --endpoint=$(CSI_ENDPOINT) + - --statedir=$(STATE_DIR) + - --storagepath=$(STORAGE_POOL) + - --nodeid=$(KUBE_NODE_NAME) + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: STORAGE_POOL + value: /tmp/kata-directvol-storages + - name: STATE_DIR + value: /csi-persist-data + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + securityContext: + privileged: true + ports: + - containerPort: 9898 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /var/lib/kubelet/plugins + mountPropagation: Bidirectional + name: plugins-dir + - mountPath: /csi-persist-data + name: csi-persist-data + - mountPath: /dev + name: dev-dir + # backend block file stored at storage-pool + - mountPath: /tmp/kata-directvol-storages + name: storage-pool + # direct volume mountInfo.json stored at shared-directvols + - mountPath: /run/kata-containers/shared/direct-volumes + name: shared-directvols + + - name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + image: registry.k8s.io/sig-storage/livenessprobe:v2.8.0 + args: + - --csi-address=/csi/csi.sock + - --health-port=9898 + + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-kata-directvolume + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins + type: Directory + name: plugins-dir + - hostPath: + # 'path' is where PV data is persisted on host. + # using /tmp is also possible while the PVs will not available after plugin container recreation or host reboot + path: /var/lib/csi-directvolume-data/ + type: DirectoryOrCreate + name: csi-persist-data + - hostPath: + path: /dev + type: Directory + name: dev-dir + # kata-containers backend rawblock stored there. + - hostPath: + path: /tmp/kata-directvol-storages + type: DirectoryOrCreate + name: storage-pool + # kata-containers direct volumes stored there. + - hostPath: + path: /run/kata-containers/shared/direct-volumes/ + type: DirectoryOrCreate + name: shared-directvols + diff --git a/src/tools/csi-kata-directvolume/deploy/rbac-deploy.sh b/src/tools/csi-kata-directvolume/deploy/rbac-deploy.sh new file mode 100755 index 0000000000..4fc1d01ee9 --- /dev/null +++ b/src/tools/csi-kata-directvolume/deploy/rbac-deploy.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# +# Copyright 2017 The Kubernetes Authors. +# Copyright (c) 2023 Ant Group +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e +set -o pipefail + +BASE_DIR=$(dirname "$0") +DEPLOY_DIR=${BASE_DIR}/kata-directvolume + +TEMP_DIR="$( mktemp -d )" +trap 'rm -rf ${TEMP_DIR}' EXIT + +: ${UPDATE_RBAC_RULES:=true} +function rbac_version () { + yaml="$1" + image="$2" + update_rbac="$3" + + # get version from `image: quay.io/k8scsi/csi-attacher:v1.0.1`, ignoring comments + version="$(sed -e 's/ *#.*$//' "$yaml" | grep "image:.*$image" | sed -e 's/ *#.*//' -e 's/.*://')" + + if $update_rbac; then + # apply overrides + varname=$(echo $image | tr - _ | tr a-z A-Z) + eval version=\${${varname}_TAG:-\${IMAGE_TAG:-\$version}} + fi + + echo "$version" +} + +# https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/${VERSION}/deploy/kubernetes/rbac.yaml +CSI_PROVISIONER_RBAC_YAML="https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/$(rbac_version "${BASE_DIR}/kata-directvolume/csi-directvol-plugin.yaml" csi-provisioner false)/deploy/kubernetes/rbac.yaml" +: ${CSI_PROVISIONER_RBAC:=https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/$(rbac_version "${BASE_DIR}/kata-directvolume/csi-directvol-plugin.yaml" csi-provisioner "${UPDATE_RBAC_RULES}")/deploy/kubernetes/rbac.yaml} + +run () { + echo "$@" >&2 + "$@" +} + +# namespace kata-directvolume +DIRECTVOL_NAMESPACE="kata-directvolume" + +# create namespace kata-directvolume +echo "Creating Namespace kata-directvolume ..." + cat <<- EOF > "${TEMP_DIR}"/kata-directvol-ns.yaml +apiVersion: v1 +kind: Namespace +metadata: + labels: + kubernetes.io/metadata.name: ${DIRECTVOL_NAMESPACE} + name: ${DIRECTVOL_NAMESPACE} +spec: + finalizers: + - kubernetes +EOF + +run kubectl apply -f "${TEMP_DIR}"/kata-directvol-ns.yaml +echo "Namespace kata-directvolume created Done !" + +# rbac rules +echo "Applying RBAC rules ..." + +eval component="CSI_PROVISIONER" +eval current="\${${component}_RBAC}" +eval original="\${${component}_RBAC_YAML}" + +if [[ "${current}" =~ ^http:// ]] || [[ "${current}" =~ ^https:// ]]; then + run curl "${current}" --output "${TEMP_DIR}"/rbac.yaml --silent --location +fi + +# replace the default namespace with specified namespace kata-directvolume +sed -e "s/namespace: default/namespace: kata-directvolume/g" "${TEMP_DIR}"/rbac.yaml > "${DEPLOY_DIR}/kata-directvol-rbac.yaml" + +# apply the kata-directvol-rbac.yaml +run kubectl apply -f "${DEPLOY_DIR}/kata-directvol-rbac.yaml" +echo "Applying RBAC rules Done!" \ No newline at end of file diff --git a/src/tools/csi-kata-directvolume/docs/deploy-csi-kata-directvol.md b/src/tools/csi-kata-directvolume/docs/deploy-csi-kata-directvol.md index 2baaac9802..c57eda4480 100644 --- a/src/tools/csi-kata-directvolume/docs/deploy-csi-kata-directvol.md +++ b/src/tools/csi-kata-directvolume/docs/deploy-csi-kata-directvol.md @@ -38,29 +38,38 @@ rolebinding.rbac.authorization.k8s.io/csi-provisioner-role-cfg created $ ./directvol-deploy.sh deploying kata directvolume components - ./csi-directvol-driverinfo.yaml + ./kata-directvolume/csi-directvol-driverinfo.yaml csidriver.storage.k8s.io/directvolume.csi.katacontainers.io created - ./csi-directvol-plugin.yaml - using image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0 - using image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0 - using image: localhost/kata-directvolume:v1.0.18 - using image: registry.k8s.io/sig-storage/livenessprobe:v2.8.0 + ./kata-directvolume/csi-directvol-plugin.yaml +kata-directvolume plugin using image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0 +kata-directvolume plugin using image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0 +kata-directvolume plugin using image: localhost/kata-directvolume:v1.0.52 +kata-directvolume plugin using image: registry.k8s.io/sig-storage/livenessprobe:v2.8.0 daemonset.apps/csi-kata-directvol-plugin created -NAMESPACE NAME READY STATUS RESTARTS AGE -default pod/csi-kata-directvol-plugin-dlphw 4/4 Running 0 3s -kube-flannel pod/kube-flannel-ds-vq796 1/1 Running 1 (52d ago) 52d -kube-system pod/coredns-66f779496c-9bmp2 1/1 Running 3 (52d ago) 52d -kube-system pod/coredns-66f779496c-qlq6d 1/1 Running 1 (52d ago) 52d -kube-system pod/etcd-node001 1/1 Running 19 (52d ago) 52d -kube-system pod/kube-apiserver-node001 1/1 Running 5 (52d ago) 52d -kube-system pod/kube-controller-manager-node001 1/1 Running 8 (52d ago) 52d -kube-system pod/kube-proxy-p9t6t 1/1 Running 6 (52d ago) 52d -kube-system pod/kube-scheduler-node001 1/1 Running 8 (52d ago) 52d + ./kata-directvolume/kata-directvol-ns.yaml +namespace/kata-directvolume unchanged + ./kata-directvolume/kata-directvol-rbac.yaml +serviceaccount/csi-provisioner unchanged +clusterrole.rbac.authorization.k8s.io/external-provisioner-runner configured +clusterrolebinding.rbac.authorization.k8s.io/csi-provisioner-role unchanged +role.rbac.authorization.k8s.io/external-provisioner-cfg unchanged +rolebinding.rbac.authorization.k8s.io/csi-provisioner-role-cfg unchanged +NAMESPACE NAME READY STATUS RESTARTS AGE +default pod/kata-driectvol-01 1/1 Running 0 3h57m +kata-directvolume pod/csi-kata-directvol-plugin-92smp 4/4 Running 0 4s +kube-flannel pod/kube-flannel-ds-vq796 1/1 Running 1 (67d ago) 67d +kube-system pod/coredns-66f779496c-9bmp2 1/1 Running 3 (67d ago) 67d +kube-system pod/coredns-66f779496c-qlq6d 1/1 Running 1 (67d ago) 67d +kube-system pod/etcd-tnt001 1/1 Running 19 (67d ago) 67d +kube-system pod/kube-apiserver-tnt001 1/1 Running 5 (67d ago) 67d +kube-system pod/kube-controller-manager-tnt001 1/1 Running 8 (67d ago) 67d +kube-system pod/kube-proxy-p9t6t 1/1 Running 6 (67d ago) 67d +kube-system pod/kube-scheduler-tnt001 1/1 Running 8 (67d ago) 67d -NAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE -default daemonset.apps/csi-kata-directvol-plugin 1 1 1 1 1 3s -kube-flannel daemonset.apps/kube-flannel-ds 1 1 1 1 1 52d -kube-system daemonset.apps/kube-proxy 1 1 1 1 1 kubernetes.io/os=linux 52d +NAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE +kata-directvolume daemonset.apps/csi-kata-directvol-plugin 1 1 1 1 1 4s +kube-flannel daemonset.apps/kube-flannel-ds 1 1 1 1 1 67d +kube-system daemonset.apps/kube-proxy 1 1 1 1 1 kubernetes.io/os=linux 67d ```