Merge pull request #12587 from fidencio/topic/runtime-add-configurable-kubelet-root-dir

runtimes: add configurable kubelet root dir

src/runtime/Makefile

@@ -305,9 +305,11 @@ DEFDANCONF := /run/kata-containers/dans
 
 DEFFORCEGUESTPULL := false
 
+DEFKUBELETROOTDIR := /var/lib/kubelet
+
 # Device cold plug
 DEFPODRESOURCEAPISOCK := ""
-DEFPODRESOURCEAPISOCK_NV := "/var/lib/kubelet/pod-resources/kubelet.sock"
+DEFPODRESOURCEAPISOCK_NV := "$(DEFKUBELETROOTDIR)/pod-resources/kubelet.sock"
 
 SED = sed
 
@@ -795,6 +797,7 @@ USER_VARS += DEFSTATICRESOURCEMGMT_NV
 USER_VARS += DEFBINDMOUNTS
 USER_VARS += DEFCREATECONTAINERTIMEOUT
 USER_VARS += DEFDANCONF
+USER_VARS += DEFKUBELETROOTDIR
 USER_VARS += DEFFORCEGUESTPULL
 USER_VARS += DEFVFIOMODE
 USER_VARS += DEFVFIOMODE_SE

src/runtime/config/configuration-clh.toml.in

@@ -491,6 +491,11 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 # (default: /run/kata-containers/dans)
 dan_conf = "@DEFDANCONF@"
 
+# kubelet_root_dir is the kubelet root directory used to match ConfigMap/Secret
+# volume paths for propagation. Override for distros that use a different path
+# (e.g. k0s: /var/lib/k0s/kubelet).
+kubelet_root_dir = "@DEFKUBELETROOTDIR@"
+
 # pod_resource_api_sock specifies the unix socket for the Kubelet's
 # PodResource API endpoint. If empty, kubernetes based cold plug
 # will not be attempted. In order for this feature to work, the

src/runtime/config/configuration-fc.toml.in

@@ -382,6 +382,11 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 # (default: /run/kata-containers/dans)
 dan_conf = "@DEFDANCONF@"
 
+# kubelet_root_dir is the kubelet root directory used to match ConfigMap/Secret
+# volume paths for propagation. Override for distros that use a different path
+# (e.g. k0s: /var/lib/k0s/kubelet).
+kubelet_root_dir = "@DEFKUBELETROOTDIR@"
+
 # pod_resource_api_sock specifies the unix socket for the Kubelet's
 # PodResource API endpoint. If empty, kubernetes based cold plug
 # will not be attempted. In order for this feature to work, the

src/runtime/config/configuration-qemu-cca.toml.in

@@ -670,6 +670,12 @@ dan_conf = "@DEFDANCONF@"
 # the container image should be pulled in the guest, without using an external snapshotter.
 # This is an experimental feature and might be removed in the future.
 experimental_force_guest_pull = @DEFFORCEGUESTPULL@
+
+# kubelet_root_dir is the kubelet root directory used to match ConfigMap/Secret
+# volume paths for propagation. Override for distros that use a different path
+# (e.g. k0s: /var/lib/k0s/kubelet).
+kubelet_root_dir = "@DEFKUBELETROOTDIR@"
+
 # pod_resource_api_sock specifies the unix socket for the Kubelet's
 # PodResource API endpoint. If empty, kubernetes based cold plug
 # will not be attempted. In order for this feature to work, the

src/runtime/config/configuration-qemu-coco-dev.toml.in

@@ -734,6 +734,11 @@ dan_conf = "@DEFDANCONF@"
 # This is an experimental feature and might be removed in the future.
 experimental_force_guest_pull = @DEFFORCEGUESTPULL@
 
+# kubelet_root_dir is the kubelet root directory used to match ConfigMap/Secret
+# volume paths for propagation. Override for distros that use a different path
+# (e.g. k0s: /var/lib/k0s/kubelet).
+kubelet_root_dir = "@DEFKUBELETROOTDIR@"
+
 # pod_resource_api_sock specifies the unix socket for the Kubelet's
 # PodResource API endpoint. If empty, kubernetes based cold plug
 # will not be attempted. In order for this feature to work, the

src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in

@@ -750,6 +750,11 @@ dan_conf = "@DEFDANCONF@"
 # This is an experimental feature and might be removed in the future.
 experimental_force_guest_pull = @DEFFORCEGUESTPULL@
 
+# kubelet_root_dir is the kubelet root directory used to match ConfigMap/Secret
+# volume paths for propagation. Override for distros that use a different path
+# (e.g. k0s: /var/lib/k0s/kubelet).
+kubelet_root_dir = "@DEFKUBELETROOTDIR@"
+
 # pod_resource_api_sock specifies the unix socket for the Kubelet's
 # PodResource API endpoint. If empty, kubernetes based cold plug
 # will not be attempted. In order for this feature to work, the

src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in

@@ -727,6 +727,11 @@ dan_conf = "@DEFDANCONF@"
 # This is an experimental feature and might be removed in the future.
 experimental_force_guest_pull = @DEFFORCEGUESTPULL@
 
+# kubelet_root_dir is the kubelet root directory used to match ConfigMap/Secret
+# volume paths for propagation. Override for distros that use a different path
+# (e.g. k0s: /var/lib/k0s/kubelet).
+kubelet_root_dir = "@DEFKUBELETROOTDIR@"
+
 # pod_resource_api_sock specifies the unix socket for the Kubelet's
 # PodResource API endpoint. If empty, kubernetes based cold plug
 # will not be attempted. In order for this feature to work, the

src/runtime/config/configuration-qemu-nvidia-gpu.toml.in

@@ -724,6 +724,11 @@ create_container_timeout = @DEFAULTTIMEOUT_NV@
 # (default: /run/kata-containers/dans)
 dan_conf = "@DEFDANCONF@"
 
+# kubelet_root_dir is the kubelet root directory used to match ConfigMap/Secret
+# volume paths for propagation. Override for distros that use a different path
+# (e.g. k0s: /var/lib/k0s/kubelet).
+kubelet_root_dir = "@DEFKUBELETROOTDIR@"
+
 # pod_resource_api_sock specifies the unix socket for the Kubelet's
 # PodResource API endpoint. If empty, kubernetes based cold plug
 # will not be attempted. In order for this feature to work, the

src/runtime/config/configuration-qemu-se.toml.in

@@ -712,6 +712,11 @@ dan_conf = "@DEFDANCONF@"
 # This is an experimental feature and might be removed in the future.
 experimental_force_guest_pull = @DEFFORCEGUESTPULL@
 
+# kubelet_root_dir is the kubelet root directory used to match ConfigMap/Secret
+# volume paths for propagation. Override for distros that use a different path
+# (e.g. k0s: /var/lib/k0s/kubelet).
+kubelet_root_dir = "@DEFKUBELETROOTDIR@"
+
 # pod_resource_api_sock specifies the unix socket for the Kubelet's
 # PodResource API endpoint. If empty, kubernetes based cold plug
 # will not be attempted. In order for this feature to work, the

src/runtime/config/configuration-qemu-snp.toml.in

@@ -737,6 +737,11 @@ dan_conf = "@DEFDANCONF@"
 # This is an experimental feature and might be removed in the future.
 experimental_force_guest_pull = @DEFFORCEGUESTPULL@
 
+# kubelet_root_dir is the kubelet root directory used to match ConfigMap/Secret
+# volume paths for propagation. Override for distros that use a different path
+# (e.g. k0s: /var/lib/k0s/kubelet).
+kubelet_root_dir = "@DEFKUBELETROOTDIR@"
+
 # pod_resource_api_sock specifies the unix socket for the Kubelet's
 # PodResource API endpoint. If empty, kubernetes based cold plug
 # will not be attempted. In order for this feature to work, the

src/runtime/config/configuration-qemu-tdx.toml.in

@@ -719,6 +719,11 @@ dan_conf = "@DEFDANCONF@"
 # This is an experimental feature and might be removed in the future.
 experimental_force_guest_pull = @DEFFORCEGUESTPULL@
 
+# kubelet_root_dir is the kubelet root directory used to match ConfigMap/Secret
+# volume paths for propagation. Override for distros that use a different path
+# (e.g. k0s: /var/lib/k0s/kubelet).
+kubelet_root_dir = "@DEFKUBELETROOTDIR@"
+
 # pod_resource_api_sock specifies the unix socket for the Kubelet's
 # PodResource API endpoint. If empty, kubernetes based cold plug
 # will not be attempted. In order for this feature to work, the

src/runtime/config/configuration-qemu.toml.in

@@ -723,6 +723,11 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 # (default: /run/kata-containers/dans)
 dan_conf = "@DEFDANCONF@"
 
+# kubelet_root_dir is the kubelet root directory used to match ConfigMap/Secret
+# volume paths for propagation. Override for distros that use a different path
+# (e.g. k0s: /var/lib/k0s/kubelet).
+kubelet_root_dir = "@DEFKUBELETROOTDIR@"
+
 # pod_resource_api_sock specifies the unix socket for the Kubelet's
 # PodResource API endpoint. If empty, kubernetes based cold plug
 # will not be attempted. In order for this feature to work, the

src/runtime/config/configuration-remote.toml.in

@@ -290,6 +290,11 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 # (default: /run/kata-containers/dans)
 dan_conf = "@DEFDANCONF@"
 
+# kubelet_root_dir is the kubelet root directory used to match ConfigMap/Secret
+# volume paths for propagation. Override for distros that use a different path
+# (e.g. k0s: /var/lib/k0s/kubelet).
+kubelet_root_dir = "@DEFKUBELETROOTDIR@"
+
 # pod_resource_api_sock specifies the unix socket for the Kubelet's
 # PodResource API endpoint. If empty, kubernetes based cold plug
 # will not be attempted. In order for this feature to work, the

src/runtime/config/configuration-stratovirt.toml.in

@@ -425,6 +425,11 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 # (default: /run/kata-containers/dans)
 dan_conf = "@DEFDANCONF@"
 
+# kubelet_root_dir is the kubelet root directory used to match ConfigMap/Secret
+# volume paths for propagation. Override for distros that use a different path
+# (e.g. k0s: /var/lib/k0s/kubelet).
+kubelet_root_dir = "@DEFKUBELETROOTDIR@"
+
 # pod_resource_api_sock specifies the unix socket for the Kubelet's
 # PodResource API endpoint. If empty, kubernetes based cold plug
 # will not be attempted. In order for this feature to work, the

src/runtime/pkg/katautils/config.go

@@ -201,6 +201,7 @@ type runtime struct {
 	DanConf                   string   `toml:"dan_conf"`
 	ForceGuestPull            bool     `toml:"experimental_force_guest_pull"`
 	PodResourceAPISock        string   `toml:"pod_resource_api_sock"`
+	KubeletRootDir            string   `toml:"kubelet_root_dir"`
 }
 
 type agent struct {
@@ -1642,6 +1643,7 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
 
 	config.ForceGuestPull = tomlConf.Runtime.ForceGuestPull
 	config.PodResourceAPISock = tomlConf.Runtime.PodResourceAPISock
+	config.KubeletRootDir = tomlConf.Runtime.KubeletRootDir
 
 	return resolved, config, nil
 }

src/runtime/pkg/oci/utils.go

@@ -193,6 +193,10 @@ type RuntimeConfig struct {
 	//	ColdPlugVFIO != NoPort AND PodResourceAPISock != "" => kubelet
 	//		based cold plug.
 	PodResourceAPISock string
+
+	// KubeletRootDir is the kubelet root directory used to match ConfigMap/Secret
+	// volume paths (e.g. /var/lib/k0s/kubelet for k0s). If empty, default is used.
+	KubeletRootDir string
 }
 
 // AddKernelParam allows the addition of new kernel parameters to an existing
@@ -1216,6 +1220,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st
 		CreateContainerTimeout: runtime.CreateContainerTimeout,
 
 		ForceGuestPull: runtime.ForceGuestPull,
+
+		KubeletRootDir: runtime.KubeletRootDir,
 	}
 
 	if err := addAnnotations(ocispec, &sandboxConfig, runtime); err != nil {

src/runtime/virtcontainers/fs_share_linux.go

@@ -58,19 +58,20 @@ func unmountNoFollow(path string) error {
 	return syscall.Unmount(path, syscall.MNT_DETACH|UmountNoFollow)
 }
 
-// Resolve the K8S root dir if it is a symbolic link
-func resolveRootDir() string {
-	rootDir, err := os.Readlink(defaultKubernetesRootDir)
-	if err != nil {
-		// Use the default root dir in case of any errors resolving the root dir symlink
-		return defaultKubernetesRootDir
+// resolveRootDirWithBase returns the resolved (followed symlink) kubelet root path.
+// If base is non-empty it is used as the root; otherwise defaultKubernetesRootDir is used.
+func resolveRootDirWithBase(base string) string {
+	if base == "" {
+		base = defaultKubernetesRootDir
+	}
+	rootDir, err := os.Readlink(base)
+	if err != nil {
+		return base
 	}
-	// Make root dir an absolute path if needed
 	if !filepath.IsAbs(rootDir) {
-		rootDir, err = filepath.Abs(filepath.Join(filepath.Dir(defaultKubernetesRootDir), rootDir))
+		rootDir, err = filepath.Abs(filepath.Join(filepath.Dir(base), rootDir))
 		if err != nil {
-			// Use the default root dir in case of any errors resolving the root dir symlink
-			return defaultKubernetesRootDir
+			return base
 		}
 	}
 	return rootDir
@@ -99,9 +100,14 @@ func NewFilesystemShare(s *Sandbox) (*FilesystemShare, error) {
 		return nil, fmt.Errorf("Creating watcher returned error %w", err)
 	}
 
-	kubernetesRootDir := resolveRootDir()
-	configVolRegex := regexp.MustCompile("^" + kubernetesRootDir + configVolRegexString)
-	timestampDirRegex := regexp.MustCompile("^" + kubernetesRootDir + configVolRegexString + timestampDirRegexString)
+	baseRoot := ""
+	if s.config != nil {
+		baseRoot = s.config.KubeletRootDir
+	}
+	kubernetesRootDir := resolveRootDirWithBase(baseRoot)
+	quotedRoot := regexp.QuoteMeta(kubernetesRootDir)
+	configVolRegex := regexp.MustCompile("^" + quotedRoot + configVolRegexString)
+	timestampDirRegex := regexp.MustCompile("^" + quotedRoot + configVolRegexString + timestampDirRegexString)
 
 	return &FilesystemShare{
 		prepared:           false,

src/runtime/virtcontainers/sandbox.go

@@ -189,6 +189,11 @@ type SandboxConfig struct {
 
 	// ForceGuestPull enforces guest pull independent of snapshotter annotations.
 	ForceGuestPull bool
+
+	// KubeletRootDir is the kubelet root directory (e.g. /var/lib/kubelet or
+	// /var/lib/k0s/kubelet for k0s). If empty, the runtime uses the default
+	// /var/lib/kubelet for matching ConfigMap/Secret volume paths.
+	KubeletRootDir string
 }
 
 // valid checks that the sandbox configuration is valid.