diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index a9c8358988..cdef157687 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -191,12 +191,8 @@ function deploy_kata() { fi if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then - yq -i \ - '.spec.template.spec.containers[0].env[6].value = "initrd kernel default_vcpus"' \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" - yq -i \ - ".spec.template.spec.containers[0].env += [{\"name\": \"HOST_OS\", \"value\": \"${KATA_HOST_OS}\"}]" \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + ALLOWED_HYPERVISOR_ANNOTATIONS="initrd kernel default_vcpus" + HOST_OS=${KATA_HOST_OS} fi if [ "${KATA_HYPERVISOR}" = "qemu" ]; then diff --git a/tools/packaging/kata-deploy/helm-chart/Makefile b/tools/packaging/kata-deploy/helm-chart/Makefile new file mode 100644 index 0000000000..00a861bcd2 --- /dev/null +++ b/tools/packaging/kata-deploy/helm-chart/Makefile @@ -0,0 +1,33 @@ +# Copyright (c) 2024 NVIDIA Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +VERSION_FILE := ../../../../VERSION + +release: + sed -i 's/appVersion: .*/appVersion: $(shell cat $(VERSION_FILE))/g' kata-deploy/Chart.yaml +all: package + +package: helm release + $(HELM) package ./kata-deploy + +.PHONY: clean + +clean: + rm kata-deploy-*.tgz + +# go-get-tool will 'go get' any package $2 and install it to $1. +PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) +define go-get-tool +@[ -f $(PROJECT_DIR)/bin/$(1) ] || command -v $(1) >/dev/null 2>&1 || { \ +set -e ;\ +echo "Downloading $(2)" ;\ +GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\ +rm -rf $$TMP_DIR ;\ +} +endef + +HELM = $(shell pwd)/bin/helm +helm: ## Download helm locally if necessary. + $(call go-get-tool,$(HELM),helm.sh/helm/v3/cmd/helm@latest) diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/.helmignore b/tools/packaging/kata-deploy/helm-chart/kata-deploy/.helmignore new file mode 100644 index 0000000000..5a0ddc5003 --- /dev/null +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/.helmignore @@ -0,0 +1,28 @@ +# Copyright (c) 2024 NVIDIA Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/Chart.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/Chart.yaml new file mode 100644 index 0000000000..919f8b673c --- /dev/null +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: kata-deploy +description: A Helm chart for deploying Kata Containers + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: "3.6.0" + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: 3.6.0 diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl new file mode 100644 index 0000000000..516b3b7056 --- /dev/null +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl @@ -0,0 +1,20 @@ +# Copyright (c) 2024 NVIDIA Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +{{/* +Set the correct containerd conf path depending on the k8s distribution +*/}} +{{- define "containerdConfPath" -}} +{{- if eq .k8sDistribution "k8s" -}} +/etc/containerd/ +{{- else if eq .k8sDistribution "rke2" -}} +/var/lib/rancher/rke2/agent/etc/containerd/ +{{- else if eq .k8sDistribution "k3s" -}} + /var/lib/rancher/k3s/agent/etc/containerd/ +{{- else if eq .k8sDistribution "k0s" -}} +/etc/k0s/containerd.d/ +{{- end -}} +{{- end -}} + diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml new file mode 100644 index 0000000000..0d3565da38 --- /dev/null +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml @@ -0,0 +1,92 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + name: {{ .Chart.Name }} + template: + metadata: + labels: + name: {{ .Chart.Name }} + spec: +{{- with .Values.imagePullSecrets }} + imagePullSecrets: +{{- toYaml . | nindent 6 }} +{{- end }} + serviceAccountName: {{ .Chart.Name }}-sa + hostPID: true + containers: + - name: kube-kata + image: {{ .Values.image.reference }}:{{ default .Chart.AppVersion .Values.image.tag }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + lifecycle: + preStop: + exec: + command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"] + command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install"] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DEBUG + value: {{ .Values.env.debug | quote }} + - name: SHIMS + value: {{ .Values.env.shims | quote }} + - name: DEFAULT_SHIM + value: {{ .Values.env.defaultShim | quote }} + - name: CREATE_RUNTIMECLASSES + value: {{ .Values.env.createRuntimeClasses | quote }} + - name: CREATE_DEFAULT_RUNTIMECLASS + value: {{ .Values.env.createDefaultRuntimeClass | quote }} + - name: ALLOWED_HYPERVISOR_ANNOTATIONS + value: {{ .Values.env.allowedHypervisorAnnotations | quote }} + - name: SNAPSHOTTER_HANDLER_MAPPING + value: {{ .Values.env.snapshotterHandlerMapping | quote }} + - name: AGENT_HTTPS_PROXY + value: {{ .Values.env.agentHttpsProxy | quote }} + - name: AGENT_NO_PROXY + value: {{ .Values.env.agentNoProxy | quote }} + - name: PULL_TYPE_MAPPING + value: {{ .Values.env.pullTypeMapping | quote }} +{{- with .Values.env.hostOS }} + - name: HOST_OS + value: {{ . | quote }} +{{- end }} + securityContext: + privileged: true + volumeMounts: + - name: crio-conf + mountPath: /etc/crio/ + - name: containerd-conf + mountPath: /etc/containerd/ + - name: kata-artifacts + mountPath: /opt/kata/ + - name: local-bin + mountPath: /usr/local/bin/ + - name: host + mountPath: /host/ + volumes: + - name: crio-conf + hostPath: + path: /etc/crio/ + - name: containerd-conf + hostPath: + path: '{{- template "containerdConfPath" .Values }}' + - name: kata-artifacts + hostPath: + path: /opt/kata/ + type: DirectoryOrCreate + - name: local-bin + hostPath: + path: /usr/local/bin/ + - name: host + hostPath: + path: / + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml new file mode 100644 index 0000000000..2d5ab4ddf8 --- /dev/null +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-sa + namespace: {{ .Release.Namespace }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Chart.Name }}-role +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "patch"] +- apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Chart.Name }}-rb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-role +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-sa + namespace: {{ .Release.Namespace }} diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml new file mode 100644 index 0000000000..b1f195d1f1 --- /dev/null +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml @@ -0,0 +1,19 @@ +imagePullPolicy: Always +imagePullSecrets: [] +image: + reference: quay.io/kata-containers/kata-deploy + tag: "" +# k8s-dist can be k8s, k3s, rke2, k0s +k8sDistribution: "k8s" +env: + debug: "false" + shims: "clh cloud-hypervisor dragonball fc qemu qemu-coco-dev qemu-runtime-rs qemu-sev qemu-snp qemu-tdx stratovirt qemu-nvidia-gpu qemu-nvidia-gpu-snp qemu-nvidia-gpu-tdx" + defaultShim: "qemu" + createRuntimeClasses: "false" + createDefaultRuntimeClass: "false" + allowedHypervisorAnnotations: "" + snapshotterHandlerMapping: "" + agentHttpsProxy: "" + agentNoProxy: "" + pullTypeMapping: "" + hostOS: ""