runtime-rs: add support for directvol csi deploy scripts.

Fixes: #8602

Signed-off-by: alex.lyn <alex.lyn@antgroup.com>
This commit is contained in:
alex.lyn
2023-12-13 20:18:54 +08:00
parent 25d8e83e43
commit c6d2a32146
7 changed files with 467 additions and 20 deletions

View File

@@ -0,0 +1,14 @@
#!/usr/bin/env bash
#
# Copyright (c) 2023 Ant Group
#
# SPDX-License-Identifier: Apache-2.0
#
set -e
set -o pipefail
BASE_DIR=$(dirname "$0")
${BASE_DIR}/rbac-deploy.sh
${BASE_DIR}/directvol-deploy.sh

View File

@@ -0,0 +1,115 @@
#!/usr/bin/env bash
#
# Copyright 2017 The Kubernetes Authors.
# Copyright (c) 2023 Ant Group
#
# SPDX-License-Identifier: Apache-2.0
#
set -e
set -o pipefail
BASE_DIR=$(dirname "$0")
# KUBELET_DATA_DIR can be set to replace the default /var/lib/kubelet.
# All nodes must use the same directory.
default_kubelet_data_dir=/var/lib/kubelet
: ${KUBELET_DATA_DIR:=${default_kubelet_data_dir}}
# namespace kata-directvolume
DIRECTVOL_NAMESPACE="kata-directvolume"
# Some images are not affected by *_REGISTRY/*_TAG and IMAGE_* variables.
# The default is to update unless explicitly excluded.
update_image () {
case "$1" in socat) return 1;; esac
}
run () {
echo "$@" >&2
"$@"
}
# deploy kata directvolume plugin and registrar sidecar
echo "deploying kata directvolume components"
for i in $(ls ${BASE_DIR}/kata-directvolume/csi-directvol-*.yaml | sort); do
echo " $i"
modified="$(cat "$i" | sed -e "s;${default_kubelet_data_dir}/;${KUBELET_DATA_DIR}/;" | while IFS= read -r line; do
nocomments="$(echo "$line" | sed -e 's/ *#.*$//')"
if echo "$nocomments" | grep -q '^[[:space:]]*image:[[:space:]]*'; then
# Split 'image: quay.io/k8scsi/csi-attacher:vx.y.z'
# into image (quay.io/k8scsi/csi-attacher:vx.y.z),
# registry (quay.io/k8scsi),
# name (csi-attacher),
# tag (vx.y.z).
image=$(echo "$nocomments" | sed -e 's;.*image:[[:space:]]*;;')
registry=$(echo "$image" | sed -e 's;\(.*\)/.*;\1;')
name=$(echo "$image" | sed -e 's;.*/\([^:]*\).*;\1;')
tag=$(echo "$image" | sed -e 's;.*:;;')
# Variables are with underscores and upper case.
varname=$(echo $name | tr - _ | tr a-z A-Z)
# Now replace registry and/or tag, if set as env variables.
# If not set, the replacement is the same as the original value.
# Only do this for the images which are meant to be configurable.
if update_image "$name"; then
prefix=$(eval echo \${${varname}_REGISTRY:-${IMAGE_REGISTRY:-${registry}}}/ | sed -e 's;none/;;')
if [ "$IMAGE_TAG" = "canary" ] &&
[ -f ${BASE_DIR}/canary-blacklist.txt ] &&
grep -q "^$name\$" ${BASE_DIR}/canary-blacklist.txt; then
# Ignore IMAGE_TAG=canary for this particular image because its
# canary image is blacklisted in the deployment blacklist.
suffix=$(eval echo :\${${varname}_TAG:-${tag}})
else
suffix=$(eval echo :\${${varname}_TAG:-${IMAGE_TAG:-${tag}}})
fi
line="$(echo "$nocomments" | sed -e "s;$image;${prefix}${name}${suffix};")"
fi
echo "kata-directvolume plugin using $line" >&2
fi
if ! $have_csistoragecapacity; then
line="$(echo "$line" | grep -v -e 'storageCapacity: true' -e '--enable-capacity')"
fi
echo "$line"
done)"
if ! echo "$modified" | kubectl apply -f -; then
echo "modified version of $i:"
echo "$modified"
exit 1
fi
done
wait_for_daemonset () {
retries=10
while [ $retries -ge 0 ]; do
ready=$(kubectl get -n $1 daemonset $2 -o jsonpath="{.status.numberReady}")
required=$(kubectl get -n $1 daemonset $2 -o jsonpath="{.status.desiredNumberScheduled}")
if [ $ready -gt 0 ] && [ $ready -eq $required ]; then
return 0
fi
retries=$((retries - 1))
sleep 3
done
return 1
}
# Wait until the DaemonSet is running on all nodes.
if ! wait_for_daemonset ${DIRECTVOL_NAMESPACE} csi-kata-directvol-plugin; then
echo
echo "driver not ready"
echo "Deployment:"
(set +e; set -x; kubectl describe all,role,clusterrole,rolebinding,clusterrolebinding,serviceaccount,storageclass,csidriver --all-namespaces -l app.kubernetes.io/instance=directvolume.csi.katacontainers.io)
echo
echo "Pod logs:"
kubectl get pods -l app.kubernetes.io/instance=directvolume.csi.katacontainers.io --all-namespaces -o=jsonpath='{range .items[*]}{.metadata.name}{" "}{range .spec.containers[*]}{.name}{" "}{end}{"\n"}{end}' | while read -r pod containers; do
for c in $containers; do
echo
(set +e; set -x; kubectl logs $pod $c)
done
done
exit 1
fi
kubectl get po,ds -A

View File

@@ -0,0 +1,17 @@
#!/usr/bin/env bash
#
# Copyright 2017 The Kubernetes Authors.
# Copyright (c) 2023 Ant Group
#
# SPDX-License-Identifier: Apache-2.0
#
set -e
set -o pipefail
# Deleting all the resources installed by the directvol-deploy script.
# Every resource in the driver installation has the label representing the installation instance.
# Using app.kubernetes.io/instance: directvolume.csi.katacontainers.io and app.kubernetes.io/part-of:
# csi-driver-kata-directvolume labels to identify the installation set
kubectl delete all --all-namespaces -l app.kubernetes.io/instance=directvolume.csi.katacontainers.io,app.kubernetes.io/part-of=csi-driver-kata-directvolume --wait=true
kubectl delete role,clusterrole,rolebinding,clusterrolebinding,serviceaccount,storageclass,csidriver --all-namespaces -l app.kubernetes.io/instance=directvolume.csi.katacontainers.io,app.kubernetes.io/part-of=csi-driver-kata-directvolume --wait=true

View File

@@ -0,0 +1,21 @@
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
name: directvolume.csi.katacontainers.io
labels:
app.kubernetes.io/instance: directvolume.csi.katacontainers.io
app.kubernetes.io/part-of: csi-driver-kata-directvolume
app.kubernetes.io/name: directvolume.csi.katacontainers.io
app.kubernetes.io/component: csi-driver
spec:
# Supports persistent volume.
volumeLifecycleModes:
- Persistent
# To determine at runtime which mode a volume uses, pod info.
podInfoOnMount: true
# No attacher needed.
attachRequired: false
storageCapacity: false
# Kubernetes may use fsGroup to change permissions and ownership
# of the volume to match user requested fsGroup in the pod's SecurityPolicy
fsGroupPolicy: File

View File

@@ -0,0 +1,190 @@
kind: DaemonSet
apiVersion: apps/v1
metadata:
namespace: kata-directvolume
name: csi-kata-directvol-plugin
labels:
app.kubernetes.io/instance: directvolume.csi.katacontainers.io
app.kubernetes.io/part-of: csi-driver-kata-directvolume
app.kubernetes.io/name: csi-kata-directvol-plugin
app.kubernetes.io/component: plugin
spec:
selector:
matchLabels:
app.kubernetes.io/instance: directvolume.csi.katacontainers.io
app.kubernetes.io/part-of: csi-driver-kata-directvolume
app.kubernetes.io/name: csi-kata-directvol-plugin
app.kubernetes.io/component: plugin
template:
metadata:
labels:
app.kubernetes.io/instance: directvolume.csi.katacontainers.io
app.kubernetes.io/part-of: csi-driver-kata-directvolume
app.kubernetes.io/name: csi-kata-directvol-plugin
app.kubernetes.io/component: plugin
spec:
serviceAccountName: csi-provisioner
containers:
- name: csi-provisioner
image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0
args:
- -v=3
- --csi-address=/csi/csi.sock
- --feature-gates=Topology=true
- --node-deployment=true
- --strict-topology=true
- --immediate-topology=false
- --worker-threads=5
#- --enable-capacity
#- --capacity-ownerref-level=0 # pod is owner
env:
- name: NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
securityContext:
# This is necessary only for systems with SELinux, where
# non-privileged sidecar containers cannot access unix domain socket
# created by privileged CSI driver container.
privileged: true
volumeMounts:
- mountPath: /csi
name: socket-dir
- name: node-driver-registrar
image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0
args:
- --v=3
- --csi-address=/csi/csi.sock
- --kubelet-registration-path=/var/lib/kubelet/plugins/csi-kata-directvolume/csi.sock
securityContext:
# This is necessary only for systems with SELinux, where
# non-privileged sidecar containers cannot access unix domain socket
# created by privileged CSI driver container.
privileged: true
env:
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
volumeMounts:
- mountPath: /csi
name: socket-dir
- mountPath: /registration
name: registration-dir
- mountPath: /csi-persist-data
name: csi-persist-data
- name: kata-directvolume
# build and push it into registry
image: localhost/kata-directvolume:v1.0.18
args:
- --drivername=directvolume.csi.katacontainers.io
- --v=5
- --endpoint=$(CSI_ENDPOINT)
- --statedir=$(STATE_DIR)
- --storagepath=$(STORAGE_POOL)
- --nodeid=$(KUBE_NODE_NAME)
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: STORAGE_POOL
value: /tmp/kata-directvol-storages
- name: STATE_DIR
value: /csi-persist-data
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
securityContext:
privileged: true
ports:
- containerPort: 9898
name: healthz
protocol: TCP
livenessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: healthz
initialDelaySeconds: 10
timeoutSeconds: 3
periodSeconds: 2
volumeMounts:
- mountPath: /csi
name: socket-dir
- mountPath: /var/lib/kubelet/pods
mountPropagation: Bidirectional
name: mountpoint-dir
- mountPath: /var/lib/kubelet/plugins
mountPropagation: Bidirectional
name: plugins-dir
- mountPath: /csi-persist-data
name: csi-persist-data
- mountPath: /dev
name: dev-dir
# backend block file stored at storage-pool
- mountPath: /tmp/kata-directvol-storages
name: storage-pool
# direct volume mountInfo.json stored at shared-directvols
- mountPath: /run/kata-containers/shared/direct-volumes
name: shared-directvols
- name: liveness-probe
volumeMounts:
- mountPath: /csi
name: socket-dir
image: registry.k8s.io/sig-storage/livenessprobe:v2.8.0
args:
- --csi-address=/csi/csi.sock
- --health-port=9898
volumes:
- hostPath:
path: /var/lib/kubelet/plugins/csi-kata-directvolume
type: DirectoryOrCreate
name: socket-dir
- hostPath:
path: /var/lib/kubelet/pods
type: DirectoryOrCreate
name: mountpoint-dir
- hostPath:
path: /var/lib/kubelet/plugins_registry
type: Directory
name: registration-dir
- hostPath:
path: /var/lib/kubelet/plugins
type: Directory
name: plugins-dir
- hostPath:
# 'path' is where PV data is persisted on host.
# using /tmp is also possible while the PVs will not available after plugin container recreation or host reboot
path: /var/lib/csi-directvolume-data/
type: DirectoryOrCreate
name: csi-persist-data
- hostPath:
path: /dev
type: Directory
name: dev-dir
# kata-containers backend rawblock stored there.
- hostPath:
path: /tmp/kata-directvol-storages
type: DirectoryOrCreate
name: storage-pool
# kata-containers direct volumes stored there.
- hostPath:
path: /run/kata-containers/shared/direct-volumes/
type: DirectoryOrCreate
name: shared-directvols

View File

@@ -0,0 +1,81 @@
#!/usr/bin/env bash
#
# Copyright 2017 The Kubernetes Authors.
# Copyright (c) 2023 Ant Group
#
# SPDX-License-Identifier: Apache-2.0
#
set -e
set -o pipefail
BASE_DIR=$(dirname "$0")
DEPLOY_DIR=${BASE_DIR}/kata-directvolume
TEMP_DIR="$( mktemp -d )"
trap 'rm -rf ${TEMP_DIR}' EXIT
: ${UPDATE_RBAC_RULES:=true}
function rbac_version () {
yaml="$1"
image="$2"
update_rbac="$3"
# get version from `image: quay.io/k8scsi/csi-attacher:v1.0.1`, ignoring comments
version="$(sed -e 's/ *#.*$//' "$yaml" | grep "image:.*$image" | sed -e 's/ *#.*//' -e 's/.*://')"
if $update_rbac; then
# apply overrides
varname=$(echo $image | tr - _ | tr a-z A-Z)
eval version=\${${varname}_TAG:-\${IMAGE_TAG:-\$version}}
fi
echo "$version"
}
# https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/${VERSION}/deploy/kubernetes/rbac.yaml
CSI_PROVISIONER_RBAC_YAML="https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/$(rbac_version "${BASE_DIR}/kata-directvolume/csi-directvol-plugin.yaml" csi-provisioner false)/deploy/kubernetes/rbac.yaml"
: ${CSI_PROVISIONER_RBAC:=https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/$(rbac_version "${BASE_DIR}/kata-directvolume/csi-directvol-plugin.yaml" csi-provisioner "${UPDATE_RBAC_RULES}")/deploy/kubernetes/rbac.yaml}
run () {
echo "$@" >&2
"$@"
}
# namespace kata-directvolume
DIRECTVOL_NAMESPACE="kata-directvolume"
# create namespace kata-directvolume
echo "Creating Namespace kata-directvolume ..."
cat <<- EOF > "${TEMP_DIR}"/kata-directvol-ns.yaml
apiVersion: v1
kind: Namespace
metadata:
labels:
kubernetes.io/metadata.name: ${DIRECTVOL_NAMESPACE}
name: ${DIRECTVOL_NAMESPACE}
spec:
finalizers:
- kubernetes
EOF
run kubectl apply -f "${TEMP_DIR}"/kata-directvol-ns.yaml
echo "Namespace kata-directvolume created Done !"
# rbac rules
echo "Applying RBAC rules ..."
eval component="CSI_PROVISIONER"
eval current="\${${component}_RBAC}"
eval original="\${${component}_RBAC_YAML}"
if [[ "${current}" =~ ^http:// ]] || [[ "${current}" =~ ^https:// ]]; then
run curl "${current}" --output "${TEMP_DIR}"/rbac.yaml --silent --location
fi
# replace the default namespace with specified namespace kata-directvolume
sed -e "s/namespace: default/namespace: kata-directvolume/g" "${TEMP_DIR}"/rbac.yaml > "${DEPLOY_DIR}/kata-directvol-rbac.yaml"
# apply the kata-directvol-rbac.yaml
run kubectl apply -f "${DEPLOY_DIR}/kata-directvol-rbac.yaml"
echo "Applying RBAC rules Done!"

View File

@@ -38,29 +38,38 @@ rolebinding.rbac.authorization.k8s.io/csi-provisioner-role-cfg created
$ ./directvol-deploy.sh
deploying kata directvolume components
./csi-directvol-driverinfo.yaml
./kata-directvolume/csi-directvol-driverinfo.yaml
csidriver.storage.k8s.io/directvolume.csi.katacontainers.io created
./csi-directvol-plugin.yaml
using image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0
using image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0
using image: localhost/kata-directvolume:v1.0.18
using image: registry.k8s.io/sig-storage/livenessprobe:v2.8.0
./kata-directvolume/csi-directvol-plugin.yaml
kata-directvolume plugin using image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0
kata-directvolume plugin using image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0
kata-directvolume plugin using image: localhost/kata-directvolume:v1.0.52
kata-directvolume plugin using image: registry.k8s.io/sig-storage/livenessprobe:v2.8.0
daemonset.apps/csi-kata-directvol-plugin created
NAMESPACE NAME READY STATUS RESTARTS AGE
default pod/csi-kata-directvol-plugin-dlphw 4/4 Running 0 3s
kube-flannel pod/kube-flannel-ds-vq796 1/1 Running 1 (52d ago) 52d
kube-system pod/coredns-66f779496c-9bmp2 1/1 Running 3 (52d ago) 52d
kube-system pod/coredns-66f779496c-qlq6d 1/1 Running 1 (52d ago) 52d
kube-system pod/etcd-node001 1/1 Running 19 (52d ago) 52d
kube-system pod/kube-apiserver-node001 1/1 Running 5 (52d ago) 52d
kube-system pod/kube-controller-manager-node001 1/1 Running 8 (52d ago) 52d
kube-system pod/kube-proxy-p9t6t 1/1 Running 6 (52d ago) 52d
kube-system pod/kube-scheduler-node001 1/1 Running 8 (52d ago) 52d
./kata-directvolume/kata-directvol-ns.yaml
namespace/kata-directvolume unchanged
./kata-directvolume/kata-directvol-rbac.yaml
serviceaccount/csi-provisioner unchanged
clusterrole.rbac.authorization.k8s.io/external-provisioner-runner configured
clusterrolebinding.rbac.authorization.k8s.io/csi-provisioner-role unchanged
role.rbac.authorization.k8s.io/external-provisioner-cfg unchanged
rolebinding.rbac.authorization.k8s.io/csi-provisioner-role-cfg unchanged
NAMESPACE NAME READY STATUS RESTARTS AGE
default pod/kata-driectvol-01 1/1 Running 0 3h57m
kata-directvolume pod/csi-kata-directvol-plugin-92smp 4/4 Running 0 4s
kube-flannel pod/kube-flannel-ds-vq796 1/1 Running 1 (67d ago) 67d
kube-system pod/coredns-66f779496c-9bmp2 1/1 Running 3 (67d ago) 67d
kube-system pod/coredns-66f779496c-qlq6d 1/1 Running 1 (67d ago) 67d
kube-system pod/etcd-tnt001 1/1 Running 19 (67d ago) 67d
kube-system pod/kube-apiserver-tnt001 1/1 Running 5 (67d ago) 67d
kube-system pod/kube-controller-manager-tnt001 1/1 Running 8 (67d ago) 67d
kube-system pod/kube-proxy-p9t6t 1/1 Running 6 (67d ago) 67d
kube-system pod/kube-scheduler-tnt001 1/1 Running 8 (67d ago) 67d
NAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
default daemonset.apps/csi-kata-directvol-plugin 1 1 1 1 1 <none> 3s
kube-flannel daemonset.apps/kube-flannel-ds 1 1 1 1 1 <none> 52d
kube-system daemonset.apps/kube-proxy 1 1 1 1 1 kubernetes.io/os=linux 52d
NAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
kata-directvolume daemonset.apps/csi-kata-directvol-plugin 1 1 1 1 1 <none> 4s
kube-flannel daemonset.apps/kube-flannel-ds 1 1 1 1 1 <none> 67d
kube-system daemonset.apps/kube-proxy 1 1 1 1 1 kubernetes.io/os=linux 67d
```