diff --git a/tests/containerd-config-v3.tmpl b/tests/containerd-config-v3.tmpl new file mode 100644 index 0000000000..d7e94c9d31 --- /dev/null +++ b/tests/containerd-config-v3.tmpl @@ -0,0 +1,124 @@ +# Copyright (c) K3s contributors +# +# SPDX-License-Identifier: Apache-2.0 +# + +{{- /* */ -}} +# File generated by {{ .Program }}. DO NOT EDIT. Use config-v3.toml.tmpl instead. +version = 3 +imports = ["__CONTAINERD_IMPORTS_PATH__"] +root = {{ printf "%q" .NodeConfig.Containerd.Root }} +state = {{ printf "%q" .NodeConfig.Containerd.State }} + +[grpc] + address = {{ deschemify .NodeConfig.Containerd.Address | printf "%q" }} + +[plugins.'io.containerd.internal.v1.opt'] + path = {{ printf "%q" .NodeConfig.Containerd.Opt }} + +[plugins.'io.containerd.grpc.v1.cri'] + stream_server_address = "127.0.0.1" + stream_server_port = "10010" + +[plugins.'io.containerd.cri.v1.runtime'] + enable_selinux = {{ .NodeConfig.SELinux }} + enable_unprivileged_ports = {{ .EnableUnprivileged }} + enable_unprivileged_icmp = {{ .EnableUnprivileged }} + device_ownership_from_security_context = {{ .NonrootDevices }} + +{{ if .DisableCgroup}} + disable_cgroup = true +{{ end }} + +{{ if .IsRunningInUserNS }} + disable_apparmor = true + restrict_oom_score_adj = true +{{ end }} + +{{ with .NodeConfig.AgentConfig.Snapshotter }} +[plugins.'io.containerd.cri.v1.images'] + snapshotter = "{{ . }}" + disable_snapshot_annotations = {{ if eq . "stargz" }}false{{else}}true{{end}} + use_local_image_pull = true +{{ end }} + +{{ with .NodeConfig.AgentConfig.PauseImage }} +[plugins.'io.containerd.cri.v1.images'.pinned_images] + sandbox = "{{ . }}" +{{ end }} + +{{- if or .NodeConfig.AgentConfig.CNIBinDir .NodeConfig.AgentConfig.CNIConfDir }} +[plugins.'io.containerd.cri.v1.runtime'.cni] + {{ with .NodeConfig.AgentConfig.CNIBinDir }}bin_dirs = [{{ printf "%q" . }}]{{ end }} + {{ with .NodeConfig.AgentConfig.CNIConfDir }}conf_dir = {{ printf "%q" . }}{{ end }} +{{ end }} + +{{ if or .NodeConfig.Containerd.BlockIOConfig .NodeConfig.Containerd.RDTConfig }} +[plugins.'io.containerd.service.v1.tasks-service'] + {{ with .NodeConfig.Containerd.BlockIOConfig }}blockio_config_file = {{ printf "%q" . }}{{ end }} + {{ with .NodeConfig.Containerd.RDTConfig }}rdt_config_file = {{ printf "%q" . }}{{ end }} +{{ end }} + +{{ with .NodeConfig.DefaultRuntime }} +[plugins.'io.containerd.cri.v1.runtime'.containerd] + default_runtime_name = "{{ . }}" +{{ end }} + +[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + +[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc.options] + SystemdCgroup = {{ .SystemdCgroup }} + +[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runhcs-wcow-process] + runtime_type = "io.containerd.runhcs.v1" + +{{ range $k, $v := .ExtraRuntimes }} +[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.'{{ $k }}'] + runtime_type = "{{$v.RuntimeType}}" +{{ with $v.BinaryName}} +[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.'{{ $k }}'.options] + BinaryName = {{ printf "%q" . }} + SystemdCgroup = {{ $.SystemdCgroup }} +{{ end }} +{{ end }} + +[plugins.'io.containerd.cri.v1.images'.registry] + config_path = {{ printf "%q" .NodeConfig.Containerd.Registry }} + +{{ if .PrivateRegistryConfig }} +{{ range $k, $v := .PrivateRegistryConfig.Configs }} +{{ with $v.Auth }} +[plugins.'io.containerd.cri.v1.images'.registry.configs.'{{ $k }}'.auth] + {{ with .Username }}username = {{ printf "%q" . }}{{ end }} + {{ with .Password }}password = {{ printf "%q" . }}{{ end }} + {{ with .Auth }}auth = {{ printf "%q" . }}{{ end }} + {{ with .IdentityToken }}identitytoken = {{ printf "%q" . }}{{ end }} +{{ end }} +{{ end }} +{{ end }} + +{{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }} +{{ with .NodeConfig.AgentConfig.ImageServiceSocket }} +[plugins.'io.containerd.snapshotter.v1.stargz'] + cri_keychain_image_service_path = {{ printf "%q" . }} + +[plugins.'io.containerd.snapshotter.v1.stargz'.cri_keychain] + enable_keychain = true +{{ end }} + +[plugins.'io.containerd.snapshotter.v1.stargz'.registry] + config_path = {{ printf "%q" .NodeConfig.Containerd.Registry }} + +{{ if .PrivateRegistryConfig }} +{{ range $k, $v := .PrivateRegistryConfig.Configs }} +{{ with $v.Auth }} +[plugins.'io.containerd.snapshotter.v1.stargz'.registry.configs.'{{ $k }}'.auth] + {{ with .Username }}username = {{ printf "%q" . }}{{ end }} + {{ with .Password }}password = {{ printf "%q" . }}{{ end }} + {{ with .Auth }}auth = {{ printf "%q" . }}{{ end }} + {{ with .IdentityToken }}identitytoken = {{ printf "%q" . }}{{ end }} +{{ end }} +{{ end }} +{{ end }} +{{ end }} diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 98800d2da9..1358d6e400 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -267,6 +267,36 @@ function deploy_k0s() { sudo chown "${USER}":"${USER}" ~/.kube/config } +# If the rendered containerd config (v3) does not import the drop-in dir, write +# the full V3 template (from tests/containerd-config-v3.tmpl) with the given +# import path and restart the service. +# Args: containerd_dir (e.g. /var/lib/rancher/k3s/agent/etc/containerd), service_name (e.g. k3s or rke2-server). +function _setup_containerd_v3_template_if_needed() { + local containerd_dir="$1" + local service_name="$2" + local template_file="${tests_dir}/containerd-config-v3.tmpl" + local rendered_v3="${containerd_dir}/config-v3.toml" + local imports_path="${containerd_dir}/config-v3.toml.d/*.toml" + if sudo test -f "${rendered_v3}" && sudo grep -q 'config-v3\.toml\.d' "${rendered_v3}" 2>/dev/null; then + return 0 + fi + if [[ ! -f "${template_file}" ]]; then + echo "Template not found: ${template_file}" >&2 + return 1 + fi + sudo mkdir -p "${containerd_dir}/config-v3.toml.d" + sed "s|__CONTAINERD_IMPORTS_PATH__|${imports_path}|g" "${template_file}" | sudo tee "${containerd_dir}/config-v3.toml.tmpl" > /dev/null + sudo systemctl restart "${service_name}" +} + +function setup_k3s_containerd_v3_template_if_needed() { + _setup_containerd_v3_template_if_needed "/var/lib/rancher/k3s/agent/etc/containerd" "k3s" +} + +function setup_rke2_containerd_v3_template_if_needed() { + _setup_containerd_v3_template_if_needed "/var/lib/rancher/rke2/agent/etc/containerd" "rke2-server" +} + function deploy_k3s() { # Set CRI runtime-request-timeout to 600s (same as kubeadm) for CoCo and long-running create requests. curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644 --kubelet-arg runtime-request-timeout=600s @@ -274,6 +304,9 @@ function deploy_k3s() { # This is an arbitrary value that came up from local tests sleep 120s + # If rendered config does not import the drop-in dir, write full V3 template so kata-deploy can use it. + setup_k3s_containerd_v3_template_if_needed + # Download the kubectl binary into /usr/bin and remove /usr/local/bin/kubectl # # We need to do this to avoid hitting issues like: @@ -343,6 +376,9 @@ function deploy_rke2() { # This is an arbitrary value that came up from local tests sleep 120s + # If rendered config does not import the drop-in dir, write full V3 template so kata-deploy can use it. + setup_rke2_containerd_v3_template_if_needed + # Link the kubectl binary into /usr/bin sudo ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl