Kubelet updates for Windows HostProcess Containers

pkg/kubelet/container/helpers.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
+	sc "k8s.io/kubernetes/pkg/securitycontext"
 	hashutil "k8s.io/kubernetes/pkg/util/hash"
 	"k8s.io/kubernetes/third_party/forked/golang/expansion"
 	utilsnet "k8s.io/utils/net"
@@ -310,6 +311,34 @@ func HasPrivilegedContainer(pod *v1.Pod) bool {
 	return hasPrivileged
 }
 
+// HasWindowsHostProcessContainer returns true if any of the containers in a pod are HostProcess containers.
+func HasWindowsHostProcessContainer(pod *v1.Pod) bool {
+	var hasHostProcess bool
+	podutil.VisitContainers(&pod.Spec, podutil.AllFeatureEnabledContainers(), func(c *v1.Container, containerType podutil.ContainerType) bool {
+		if sc.HasWindowsHostProcessRequest(pod, c) {
+			hasHostProcess = true
+			return false
+		}
+		return true
+	})
+
+	return hasHostProcess
+}
+
+// AllContainersAreWindowsHostProcess returns true if all containres in a pod are HostProcess containers.
+func AllContainersAreWindowsHostProcess(pod *v1.Pod) bool {
+	allHostProcess := true
+	podutil.VisitContainers(&pod.Spec, podutil.AllFeatureEnabledContainers(), func(c *v1.Container, containerType podutil.ContainerType) bool {
+		if !sc.HasWindowsHostProcessRequest(pod, c) {
+			allHostProcess = false
+			return false
+		}
+		return true
+	})
+
+	return allHostProcess
+}
+
 // MakePortMappings creates internal port mapping from api port mapping.
 func MakePortMappings(container *v1.Container) (ports []PortMapping) {
 	names := make(map[string]struct{})

pkg/kubelet/kuberuntime/kuberuntime_container_windows.go

@@ -19,14 +19,16 @@ limitations under the License.
 package kuberuntime
 
 import (
+	"fmt"
 	"runtime"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/features"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/securitycontext"
-
-	"k8s.io/klog/v2"
 )
 
 // applyPlatformSpecificContainerConfig applies platform specific configurations to runtimeapi.ContainerConfig.
@@ -122,5 +124,12 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1
 		wc.SecurityContext.RunAsUsername = *effectiveSc.WindowsOptions.RunAsUserName
 	}
 
+	if securitycontext.HasWindowsHostProcessRequest(pod, container) {
+		if !utilfeature.DefaultFeatureGate.Enabled(features.WindowsHostProcessContainers) {
+			return nil, fmt.Errorf("pod contains HostProcess containers but feature 'WindowsHostProcessContainers' is not enabled")
+		}
+		wc.SecurityContext.HostProcess = true
+	}
+
 	return wc, nil
 }

pkg/kubelet/kuberuntime/kuberuntime_sandbox.go

@@ -25,8 +25,10 @@ import (
 
 	v1 "k8s.io/api/core/v1"
 	kubetypes "k8s.io/apimachinery/pkg/types"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/features"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/kubelet/util"
@@ -138,6 +140,14 @@ func (m *kubeGenericRuntimeManager) generatePodSandboxConfig(pod *v1.Pod, attemp
 	}
 	podSandboxConfig.Linux = lc
 
+	if runtime.GOOS == "windows" {
+		wc, err := m.generatePodSandboxWindowsConfig(pod)
+		if err != nil {
+			return nil, err
+		}
+		podSandboxConfig.Windows = wc
+	}
+
 	return podSandboxConfig, nil
 }
 
@@ -206,6 +216,54 @@ func (m *kubeGenericRuntimeManager) generatePodSandboxLinuxConfig(pod *v1.Pod) (
 	return lc, nil
 }
 
+// generatePodSandboxWindowsConfig generates WindowsPodSandboxConfig from v1.Pod.
+// On Windows this will get called in addition to LinuxPodSandboxConfig because not all relevant fields have been added to
+// WindowsPodSandboxConfig at this time.
+func (m *kubeGenericRuntimeManager) generatePodSandboxWindowsConfig(pod *v1.Pod) (*runtimeapi.WindowsPodSandboxConfig, error) {
+	wc := &runtimeapi.WindowsPodSandboxConfig{
+		SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{},
+	}
+
+	sc := pod.Spec.SecurityContext
+	if sc == nil || sc.WindowsOptions == nil {
+		return wc, nil
+	}
+
+	wo := sc.WindowsOptions
+	if wo.GMSACredentialSpec != nil {
+		wc.SecurityContext.CredentialSpec = *wo.GMSACredentialSpec
+	}
+
+	if wo.RunAsUserName != nil {
+		wc.SecurityContext.RunAsUsername = *wo.RunAsUserName
+	}
+
+	if kubecontainer.HasWindowsHostProcessContainer(pod) {
+		// Pods containing HostProcess containers should fail to schedule if feature is not
+		// enabled instead of trying to schedule containers as regular containers as stated in
+		// PRR review.
+		if !utilfeature.DefaultFeatureGate.Enabled(features.WindowsHostProcessContainers) {
+			return nil, fmt.Errorf("pod contains HostProcess containers but feature 'WindowsHostProcessContainers' is not enabled")
+		}
+
+		if wo.HostProcess != nil && !*wo.HostProcess {
+			return nil, fmt.Errorf("pod must not contain any HostProcess containers if Pod's WindowsOptions.HostProcess is set to false")
+		}
+		// At present Windows all containers in a Windows pod must be HostProcess containers
+		// and HostNetwork is required to be set.
+		if !kubecontainer.AllContainersAreWindowsHostProcess(pod) {
+			return nil, fmt.Errorf("pod must not contain both HostProcess and non-HostProcess containers")
+		}
+		if !kubecontainer.IsHostNetworkPod(pod) {
+			return nil, fmt.Errorf("hostNetwork is required if Pod contains HostProcess containers")
+		}
+
+		wc.SecurityContext.HostProcess = true
+	}
+
+	return wc, nil
+}
+
 // getKubeletSandboxes lists all (or just the running) sandboxes managed by kubelet.
 func (m *kubeGenericRuntimeManager) getKubeletSandboxes(all bool) ([]*runtimeapi.PodSandbox, error) {
 	var filter *runtimeapi.PodSandboxFilter

pkg/kubelet/kuberuntime/kuberuntime_sandbox_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package kuberuntime
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
@@ -25,7 +26,10 @@ import (
 	"github.com/stretchr/testify/require"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+	"k8s.io/kubernetes/pkg/features"
 	containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
 	"k8s.io/kubernetes/pkg/kubelet/runtimeclass"
 	rctest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
@@ -172,3 +176,187 @@ func newSeccompPod(podFieldProfile, containerFieldProfile *v1.SeccompProfile, po
 	}
 	return pod
 }
+
+func TestGeneratePodSandboxWindowsConfig(t *testing.T) {
+	_, _, m, err := createTestRuntimeManager()
+	require.NoError(t, err)
+
+	const containerName = "container"
+	gmsaCreds := "gmsa-creds"
+	userName := "SYSTEM"
+	trueVar := true
+	falseVar := false
+
+	testCases := []struct {
+		name                      string
+		hostProcessFeatureEnabled bool
+		podSpec                   *v1.PodSpec
+		expectedWindowsConfig     *runtimeapi.WindowsPodSandboxConfig
+		expectedError             error
+	}{
+		{
+			name:                      "Empty PodSecurityContext",
+			hostProcessFeatureEnabled: false,
+			podSpec: &v1.PodSpec{
+				Containers: []v1.Container{{
+					Name: containerName,
+				}},
+			},
+			expectedWindowsConfig: &runtimeapi.WindowsPodSandboxConfig{
+				SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{},
+			},
+			expectedError: nil,
+		},
+		{
+			name:                      "GMSACredentialSpec in PodSecurityContext",
+			hostProcessFeatureEnabled: false,
+			podSpec: &v1.PodSpec{
+				SecurityContext: &v1.PodSecurityContext{
+					WindowsOptions: &v1.WindowsSecurityContextOptions{
+						GMSACredentialSpec: &gmsaCreds,
+					},
+				},
+				Containers: []v1.Container{{
+					Name: containerName,
+				}},
+			},
+			expectedWindowsConfig: &runtimeapi.WindowsPodSandboxConfig{
+				SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{
+					CredentialSpec: "gmsa-creds",
+				},
+			},
+			expectedError: nil,
+		},
+		{
+			name:                      "RunAsUserName in PodSecurityContext",
+			hostProcessFeatureEnabled: false,
+			podSpec: &v1.PodSpec{
+				SecurityContext: &v1.PodSecurityContext{
+					WindowsOptions: &v1.WindowsSecurityContextOptions{
+						RunAsUserName: &userName,
+					},
+				},
+				Containers: []v1.Container{{
+					Name: containerName,
+				}},
+			},
+			expectedWindowsConfig: &runtimeapi.WindowsPodSandboxConfig{
+				SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{
+					RunAsUsername: "SYSTEM",
+				},
+			},
+			expectedError: nil,
+		},
+		{
+			name:                      "Pod with HostProcess containers and feature gate disabled",
+			hostProcessFeatureEnabled: false,
+			podSpec: &v1.PodSpec{
+				SecurityContext: &v1.PodSecurityContext{
+					WindowsOptions: &v1.WindowsSecurityContextOptions{
+						HostProcess: &trueVar,
+					},
+				},
+				Containers: []v1.Container{{
+					Name: containerName,
+				}},
+			},
+			expectedWindowsConfig: nil,
+			expectedError:         fmt.Errorf("pod contains HostProcess containers but feature 'WindowsHostProcessContainers' is not enabled"),
+		},
+		{
+			name:                      "Pod with HostProcess containers and non-HostProcess containers",
+			hostProcessFeatureEnabled: true,
+			podSpec: &v1.PodSpec{
+				SecurityContext: &v1.PodSecurityContext{
+					WindowsOptions: &v1.WindowsSecurityContextOptions{
+						HostProcess: &trueVar,
+					},
+				},
+				Containers: []v1.Container{{
+					Name: containerName,
+				}, {
+					Name: containerName,
+					SecurityContext: &v1.SecurityContext{
+						WindowsOptions: &v1.WindowsSecurityContextOptions{
+							HostProcess: &falseVar,
+						},
+					},
+				}},
+			},
+			expectedWindowsConfig: nil,
+			expectedError:         fmt.Errorf("pod must not contain both HostProcess and non-HostProcess containers"),
+		},
+		{
+			name:                      "Pod with HostProcess containers and HostNetwork not set",
+			hostProcessFeatureEnabled: true,
+			podSpec: &v1.PodSpec{
+				SecurityContext: &v1.PodSecurityContext{
+					WindowsOptions: &v1.WindowsSecurityContextOptions{
+						HostProcess: &trueVar,
+					},
+				},
+				Containers: []v1.Container{{
+					Name: containerName,
+				}},
+			},
+			expectedWindowsConfig: nil,
+			expectedError:         fmt.Errorf("hostNetwork is required if Pod contains HostProcess containers"),
+		},
+		{
+			name:                      "Pod with HostProcess containers and HostNetwork set",
+			hostProcessFeatureEnabled: true,
+			podSpec: &v1.PodSpec{
+				HostNetwork: true,
+				SecurityContext: &v1.PodSecurityContext{
+					WindowsOptions: &v1.WindowsSecurityContextOptions{
+						HostProcess: &trueVar,
+					},
+				},
+				Containers: []v1.Container{{
+					Name: containerName,
+				}},
+			},
+			expectedWindowsConfig: &runtimeapi.WindowsPodSandboxConfig{
+				SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{
+					HostProcess: true,
+				},
+			},
+			expectedError: nil,
+		},
+		{
+			name:                      "Pod's WindowsOptions.HostProcess set to false and pod has HostProcess containers",
+			hostProcessFeatureEnabled: true,
+			podSpec: &v1.PodSpec{
+				HostNetwork: true,
+				SecurityContext: &v1.PodSecurityContext{
+					WindowsOptions: &v1.WindowsSecurityContextOptions{
+						HostProcess: &falseVar,
+					},
+				},
+				Containers: []v1.Container{{
+					Name: containerName,
+					SecurityContext: &v1.SecurityContext{
+						WindowsOptions: &v1.WindowsSecurityContextOptions{
+							HostProcess: &trueVar,
+						},
+					},
+				}},
+			},
+			expectedWindowsConfig: nil,
+			expectedError:         fmt.Errorf("pod must not contain any HostProcess containers if Pod's WindowsOptions.HostProcess is set to false"),
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WindowsHostProcessContainers, testCase.hostProcessFeatureEnabled)()
+			pod := &v1.Pod{}
+			pod.Spec = *testCase.podSpec
+
+			wc, err := m.generatePodSandboxWindowsConfig(pod)
+
+			assert.Equal(t, wc, testCase.expectedWindowsConfig)
+			assert.Equal(t, err, testCase.expectedError)
+		})
+	}
+}

pkg/kubelet/kuberuntime/labels.go

@@ -18,13 +18,17 @@ package kuberuntime
 
 import (
 	"encoding/json"
+	"runtime"
 	"strconv"
 
 	v1 "k8s.io/api/core/v1"
 	kubetypes "k8s.io/apimachinery/pkg/types"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/features"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/types"
+	sc "k8s.io/kubernetes/pkg/securitycontext"
 )
 
 const (
@@ -38,6 +42,12 @@ const (
 	containerTerminationMessagePolicyLabel = "io.kubernetes.container.terminationMessagePolicy"
 	containerPreStopHandlerLabel           = "io.kubernetes.container.preStopHandler"
 	containerPortsLabel                    = "io.kubernetes.container.ports"
+
+	// TODO: remove this annotation when moving to beta for Windows hostprocess containers
+	// xref: https://github.com/kubernetes/kubernetes/pull/99576/commits/42fb66073214eed6fe43fa8b1586f396e30e73e3#r635392090
+	// Currently, ContainerD on Windows does not yet fully support HostProcess containers
+	// but will pass annotations to hcsshim which does have support.
+	windowsHostProcessContainer = "microsoft.com/hostprocess-container"
 )
 
 type labeledPodSandboxInfo struct {
@@ -89,7 +99,23 @@ func newPodLabels(pod *v1.Pod) map[string]string {
 
 // newPodAnnotations creates pod annotations from v1.Pod.
 func newPodAnnotations(pod *v1.Pod) map[string]string {
-	return pod.Annotations
+	annotations := map[string]string{}
+
+	// Get annotations from v1.Pod
+	for k, v := range pod.Annotations {
+		annotations[k] = v
+	}
+
+	if runtime.GOOS == "windows" && utilfeature.DefaultFeatureGate.Enabled(features.WindowsHostProcessContainers) {
+		if kubecontainer.HasWindowsHostProcessContainer(pod) {
+			// While WindowsHostProcessContainers is in alpha pass 'microsoft.com/hostprocess-container' annotation
+			// to pod sandbox creations request. ContainerD on Windows does not yet fully support HostProcess
+			// containers but will pass annotations to hcsshim which does have support.
+			annotations[windowsHostProcessContainer] = "true"
+		}
+	}
+
+	return annotations
 }
 
 // newContainerLabels creates container labels from v1.Container and v1.Pod.
@@ -143,6 +169,15 @@ func newContainerAnnotations(container *v1.Container, pod *v1.Pod, restartCount
 		}
 	}
 
+	if runtime.GOOS == "windows" && utilfeature.DefaultFeatureGate.Enabled(features.WindowsHostProcessContainers) {
+		if sc.HasWindowsHostProcessRequest(pod, container) {
+			// While WindowsHostProcessContainers is in alpha pass 'microsoft.com/hostprocess-container' annotation
+			// to create containers request. ContainerD on Windows does not yet fully support HostProcess containers
+			// but will pass annotations to hcsshim which does have support.
+			annotations[windowsHostProcessContainer] = "true"
+		}
+	}
+
 	return annotations
 }
 

pkg/securitycontext/util.go

@@ -44,6 +44,20 @@ func HasCapabilitiesRequest(container *v1.Container) bool {
 	return len(container.SecurityContext.Capabilities.Add) > 0 || len(container.SecurityContext.Capabilities.Drop) > 0
 }
 
+// HasWindowsHostProcessRequest returns true if container should run as HostProcess container,
+// taking into account nils
+func HasWindowsHostProcessRequest(pod *v1.Pod, container *v1.Container) bool {
+	effectiveSc := DetermineEffectiveSecurityContext(pod, container)
+
+	if effectiveSc.WindowsOptions == nil {
+		return false
+	}
+	if effectiveSc.WindowsOptions.HostProcess == nil {
+		return false
+	}
+	return *effectiveSc.WindowsOptions.HostProcess
+}
+
 // DetermineEffectiveSecurityContext returns a synthesized SecurityContext for reading effective configurations
 // from the provided pod's and container's security context. Container's fields take precedence in cases where both
 // are set
@@ -79,6 +93,9 @@ func DetermineEffectiveSecurityContext(pod *v1.Pod, container *v1.Container) *v1
 		if containerSc.WindowsOptions.RunAsUserName != nil {
 			effectiveSc.WindowsOptions.RunAsUserName = containerSc.WindowsOptions.RunAsUserName
 		}
+		if containerSc.WindowsOptions.HostProcess != nil {
+			effectiveSc.WindowsOptions.HostProcess = containerSc.WindowsOptions.HostProcess
+		}
 	}
 
 	if containerSc.Capabilities != nil {