diff --git a/pkg/kubelet/container/helpers.go b/pkg/kubelet/container/helpers.go index 7e3314fec01..b48ba532db0 100644 --- a/pkg/kubelet/container/helpers.go +++ b/pkg/kubelet/container/helpers.go @@ -31,6 +31,7 @@ import ( "k8s.io/client-go/tools/record" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" podutil "k8s.io/kubernetes/pkg/api/v1/pod" + sc "k8s.io/kubernetes/pkg/securitycontext" hashutil "k8s.io/kubernetes/pkg/util/hash" "k8s.io/kubernetes/third_party/forked/golang/expansion" utilsnet "k8s.io/utils/net" @@ -310,6 +311,34 @@ func HasPrivilegedContainer(pod *v1.Pod) bool { return hasPrivileged } +// HasWindowsHostProcessContainer returns true if any of the containers in a pod are HostProcess containers. +func HasWindowsHostProcessContainer(pod *v1.Pod) bool { + var hasHostProcess bool + podutil.VisitContainers(&pod.Spec, podutil.AllFeatureEnabledContainers(), func(c *v1.Container, containerType podutil.ContainerType) bool { + if sc.HasWindowsHostProcessRequest(pod, c) { + hasHostProcess = true + return false + } + return true + }) + + return hasHostProcess +} + +// AllContainersAreWindowsHostProcess returns true if all containres in a pod are HostProcess containers. +func AllContainersAreWindowsHostProcess(pod *v1.Pod) bool { + allHostProcess := true + podutil.VisitContainers(&pod.Spec, podutil.AllFeatureEnabledContainers(), func(c *v1.Container, containerType podutil.ContainerType) bool { + if !sc.HasWindowsHostProcessRequest(pod, c) { + allHostProcess = false + return false + } + return true + }) + + return allHostProcess +} + // MakePortMappings creates internal port mapping from api port mapping. func MakePortMappings(container *v1.Container) (ports []PortMapping) { names := make(map[string]struct{}) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go b/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go index 5b4910e5fbf..d295bc80931 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go @@ -19,14 +19,16 @@ limitations under the License. package kuberuntime import ( + "fmt" "runtime" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/features" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/securitycontext" - - "k8s.io/klog/v2" ) // applyPlatformSpecificContainerConfig applies platform specific configurations to runtimeapi.ContainerConfig. @@ -122,5 +124,12 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1 wc.SecurityContext.RunAsUsername = *effectiveSc.WindowsOptions.RunAsUserName } + if securitycontext.HasWindowsHostProcessRequest(pod, container) { + if !utilfeature.DefaultFeatureGate.Enabled(features.WindowsHostProcessContainers) { + return nil, fmt.Errorf("pod contains HostProcess containers but feature 'WindowsHostProcessContainers' is not enabled") + } + wc.SecurityContext.HostProcess = true + } + return wc, nil } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go b/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go index 60cc9dbb04a..95832f9f13e 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go @@ -25,8 +25,10 @@ import ( v1 "k8s.io/api/core/v1" kubetypes "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/features" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/kubelet/util" @@ -138,6 +140,14 @@ func (m *kubeGenericRuntimeManager) generatePodSandboxConfig(pod *v1.Pod, attemp } podSandboxConfig.Linux = lc + if runtime.GOOS == "windows" { + wc, err := m.generatePodSandboxWindowsConfig(pod) + if err != nil { + return nil, err + } + podSandboxConfig.Windows = wc + } + return podSandboxConfig, nil } @@ -206,6 +216,54 @@ func (m *kubeGenericRuntimeManager) generatePodSandboxLinuxConfig(pod *v1.Pod) ( return lc, nil } +// generatePodSandboxWindowsConfig generates WindowsPodSandboxConfig from v1.Pod. +// On Windows this will get called in addition to LinuxPodSandboxConfig because not all relevant fields have been added to +// WindowsPodSandboxConfig at this time. +func (m *kubeGenericRuntimeManager) generatePodSandboxWindowsConfig(pod *v1.Pod) (*runtimeapi.WindowsPodSandboxConfig, error) { + wc := &runtimeapi.WindowsPodSandboxConfig{ + SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{}, + } + + sc := pod.Spec.SecurityContext + if sc == nil || sc.WindowsOptions == nil { + return wc, nil + } + + wo := sc.WindowsOptions + if wo.GMSACredentialSpec != nil { + wc.SecurityContext.CredentialSpec = *wo.GMSACredentialSpec + } + + if wo.RunAsUserName != nil { + wc.SecurityContext.RunAsUsername = *wo.RunAsUserName + } + + if kubecontainer.HasWindowsHostProcessContainer(pod) { + // Pods containing HostProcess containers should fail to schedule if feature is not + // enabled instead of trying to schedule containers as regular containers as stated in + // PRR review. + if !utilfeature.DefaultFeatureGate.Enabled(features.WindowsHostProcessContainers) { + return nil, fmt.Errorf("pod contains HostProcess containers but feature 'WindowsHostProcessContainers' is not enabled") + } + + if wo.HostProcess != nil && !*wo.HostProcess { + return nil, fmt.Errorf("pod must not contain any HostProcess containers if Pod's WindowsOptions.HostProcess is set to false") + } + // At present Windows all containers in a Windows pod must be HostProcess containers + // and HostNetwork is required to be set. + if !kubecontainer.AllContainersAreWindowsHostProcess(pod) { + return nil, fmt.Errorf("pod must not contain both HostProcess and non-HostProcess containers") + } + if !kubecontainer.IsHostNetworkPod(pod) { + return nil, fmt.Errorf("hostNetwork is required if Pod contains HostProcess containers") + } + + wc.SecurityContext.HostProcess = true + } + + return wc, nil +} + // getKubeletSandboxes lists all (or just the running) sandboxes managed by kubelet. func (m *kubeGenericRuntimeManager) getKubeletSandboxes(all bool) ([]*runtimeapi.PodSandbox, error) { var filter *runtimeapi.PodSandboxFilter diff --git a/pkg/kubelet/kuberuntime/kuberuntime_sandbox_test.go b/pkg/kubelet/kuberuntime/kuberuntime_sandbox_test.go index abbc867349e..c7441d55b99 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_sandbox_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_sandbox_test.go @@ -17,6 +17,7 @@ limitations under the License. package kuberuntime import ( + "fmt" "os" "path/filepath" "testing" @@ -25,7 +26,10 @@ import ( "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + "k8s.io/kubernetes/pkg/features" containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" "k8s.io/kubernetes/pkg/kubelet/runtimeclass" rctest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing" @@ -172,3 +176,187 @@ func newSeccompPod(podFieldProfile, containerFieldProfile *v1.SeccompProfile, po } return pod } + +func TestGeneratePodSandboxWindowsConfig(t *testing.T) { + _, _, m, err := createTestRuntimeManager() + require.NoError(t, err) + + const containerName = "container" + gmsaCreds := "gmsa-creds" + userName := "SYSTEM" + trueVar := true + falseVar := false + + testCases := []struct { + name string + hostProcessFeatureEnabled bool + podSpec *v1.PodSpec + expectedWindowsConfig *runtimeapi.WindowsPodSandboxConfig + expectedError error + }{ + { + name: "Empty PodSecurityContext", + hostProcessFeatureEnabled: false, + podSpec: &v1.PodSpec{ + Containers: []v1.Container{{ + Name: containerName, + }}, + }, + expectedWindowsConfig: &runtimeapi.WindowsPodSandboxConfig{ + SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{}, + }, + expectedError: nil, + }, + { + name: "GMSACredentialSpec in PodSecurityContext", + hostProcessFeatureEnabled: false, + podSpec: &v1.PodSpec{ + SecurityContext: &v1.PodSecurityContext{ + WindowsOptions: &v1.WindowsSecurityContextOptions{ + GMSACredentialSpec: &gmsaCreds, + }, + }, + Containers: []v1.Container{{ + Name: containerName, + }}, + }, + expectedWindowsConfig: &runtimeapi.WindowsPodSandboxConfig{ + SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{ + CredentialSpec: "gmsa-creds", + }, + }, + expectedError: nil, + }, + { + name: "RunAsUserName in PodSecurityContext", + hostProcessFeatureEnabled: false, + podSpec: &v1.PodSpec{ + SecurityContext: &v1.PodSecurityContext{ + WindowsOptions: &v1.WindowsSecurityContextOptions{ + RunAsUserName: &userName, + }, + }, + Containers: []v1.Container{{ + Name: containerName, + }}, + }, + expectedWindowsConfig: &runtimeapi.WindowsPodSandboxConfig{ + SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{ + RunAsUsername: "SYSTEM", + }, + }, + expectedError: nil, + }, + { + name: "Pod with HostProcess containers and feature gate disabled", + hostProcessFeatureEnabled: false, + podSpec: &v1.PodSpec{ + SecurityContext: &v1.PodSecurityContext{ + WindowsOptions: &v1.WindowsSecurityContextOptions{ + HostProcess: &trueVar, + }, + }, + Containers: []v1.Container{{ + Name: containerName, + }}, + }, + expectedWindowsConfig: nil, + expectedError: fmt.Errorf("pod contains HostProcess containers but feature 'WindowsHostProcessContainers' is not enabled"), + }, + { + name: "Pod with HostProcess containers and non-HostProcess containers", + hostProcessFeatureEnabled: true, + podSpec: &v1.PodSpec{ + SecurityContext: &v1.PodSecurityContext{ + WindowsOptions: &v1.WindowsSecurityContextOptions{ + HostProcess: &trueVar, + }, + }, + Containers: []v1.Container{{ + Name: containerName, + }, { + Name: containerName, + SecurityContext: &v1.SecurityContext{ + WindowsOptions: &v1.WindowsSecurityContextOptions{ + HostProcess: &falseVar, + }, + }, + }}, + }, + expectedWindowsConfig: nil, + expectedError: fmt.Errorf("pod must not contain both HostProcess and non-HostProcess containers"), + }, + { + name: "Pod with HostProcess containers and HostNetwork not set", + hostProcessFeatureEnabled: true, + podSpec: &v1.PodSpec{ + SecurityContext: &v1.PodSecurityContext{ + WindowsOptions: &v1.WindowsSecurityContextOptions{ + HostProcess: &trueVar, + }, + }, + Containers: []v1.Container{{ + Name: containerName, + }}, + }, + expectedWindowsConfig: nil, + expectedError: fmt.Errorf("hostNetwork is required if Pod contains HostProcess containers"), + }, + { + name: "Pod with HostProcess containers and HostNetwork set", + hostProcessFeatureEnabled: true, + podSpec: &v1.PodSpec{ + HostNetwork: true, + SecurityContext: &v1.PodSecurityContext{ + WindowsOptions: &v1.WindowsSecurityContextOptions{ + HostProcess: &trueVar, + }, + }, + Containers: []v1.Container{{ + Name: containerName, + }}, + }, + expectedWindowsConfig: &runtimeapi.WindowsPodSandboxConfig{ + SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{ + HostProcess: true, + }, + }, + expectedError: nil, + }, + { + name: "Pod's WindowsOptions.HostProcess set to false and pod has HostProcess containers", + hostProcessFeatureEnabled: true, + podSpec: &v1.PodSpec{ + HostNetwork: true, + SecurityContext: &v1.PodSecurityContext{ + WindowsOptions: &v1.WindowsSecurityContextOptions{ + HostProcess: &falseVar, + }, + }, + Containers: []v1.Container{{ + Name: containerName, + SecurityContext: &v1.SecurityContext{ + WindowsOptions: &v1.WindowsSecurityContextOptions{ + HostProcess: &trueVar, + }, + }, + }}, + }, + expectedWindowsConfig: nil, + expectedError: fmt.Errorf("pod must not contain any HostProcess containers if Pod's WindowsOptions.HostProcess is set to false"), + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WindowsHostProcessContainers, testCase.hostProcessFeatureEnabled)() + pod := &v1.Pod{} + pod.Spec = *testCase.podSpec + + wc, err := m.generatePodSandboxWindowsConfig(pod) + + assert.Equal(t, wc, testCase.expectedWindowsConfig) + assert.Equal(t, err, testCase.expectedError) + }) + } +} diff --git a/pkg/kubelet/kuberuntime/labels.go b/pkg/kubelet/kuberuntime/labels.go index 2aaa1743ac6..4ee13e7337d 100644 --- a/pkg/kubelet/kuberuntime/labels.go +++ b/pkg/kubelet/kuberuntime/labels.go @@ -18,13 +18,17 @@ package kuberuntime import ( "encoding/json" + "runtime" "strconv" v1 "k8s.io/api/core/v1" kubetypes "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/features" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/types" + sc "k8s.io/kubernetes/pkg/securitycontext" ) const ( @@ -38,6 +42,12 @@ const ( containerTerminationMessagePolicyLabel = "io.kubernetes.container.terminationMessagePolicy" containerPreStopHandlerLabel = "io.kubernetes.container.preStopHandler" containerPortsLabel = "io.kubernetes.container.ports" + + // TODO: remove this annotation when moving to beta for Windows hostprocess containers + // xref: https://github.com/kubernetes/kubernetes/pull/99576/commits/42fb66073214eed6fe43fa8b1586f396e30e73e3#r635392090 + // Currently, ContainerD on Windows does not yet fully support HostProcess containers + // but will pass annotations to hcsshim which does have support. + windowsHostProcessContainer = "microsoft.com/hostprocess-container" ) type labeledPodSandboxInfo struct { @@ -89,7 +99,23 @@ func newPodLabels(pod *v1.Pod) map[string]string { // newPodAnnotations creates pod annotations from v1.Pod. func newPodAnnotations(pod *v1.Pod) map[string]string { - return pod.Annotations + annotations := map[string]string{} + + // Get annotations from v1.Pod + for k, v := range pod.Annotations { + annotations[k] = v + } + + if runtime.GOOS == "windows" && utilfeature.DefaultFeatureGate.Enabled(features.WindowsHostProcessContainers) { + if kubecontainer.HasWindowsHostProcessContainer(pod) { + // While WindowsHostProcessContainers is in alpha pass 'microsoft.com/hostprocess-container' annotation + // to pod sandbox creations request. ContainerD on Windows does not yet fully support HostProcess + // containers but will pass annotations to hcsshim which does have support. + annotations[windowsHostProcessContainer] = "true" + } + } + + return annotations } // newContainerLabels creates container labels from v1.Container and v1.Pod. @@ -143,6 +169,15 @@ func newContainerAnnotations(container *v1.Container, pod *v1.Pod, restartCount } } + if runtime.GOOS == "windows" && utilfeature.DefaultFeatureGate.Enabled(features.WindowsHostProcessContainers) { + if sc.HasWindowsHostProcessRequest(pod, container) { + // While WindowsHostProcessContainers is in alpha pass 'microsoft.com/hostprocess-container' annotation + // to create containers request. ContainerD on Windows does not yet fully support HostProcess containers + // but will pass annotations to hcsshim which does have support. + annotations[windowsHostProcessContainer] = "true" + } + } + return annotations } diff --git a/pkg/securitycontext/util.go b/pkg/securitycontext/util.go index e960dc9325f..82a2fc5e0a9 100644 --- a/pkg/securitycontext/util.go +++ b/pkg/securitycontext/util.go @@ -44,6 +44,20 @@ func HasCapabilitiesRequest(container *v1.Container) bool { return len(container.SecurityContext.Capabilities.Add) > 0 || len(container.SecurityContext.Capabilities.Drop) > 0 } +// HasWindowsHostProcessRequest returns true if container should run as HostProcess container, +// taking into account nils +func HasWindowsHostProcessRequest(pod *v1.Pod, container *v1.Container) bool { + effectiveSc := DetermineEffectiveSecurityContext(pod, container) + + if effectiveSc.WindowsOptions == nil { + return false + } + if effectiveSc.WindowsOptions.HostProcess == nil { + return false + } + return *effectiveSc.WindowsOptions.HostProcess +} + // DetermineEffectiveSecurityContext returns a synthesized SecurityContext for reading effective configurations // from the provided pod's and container's security context. Container's fields take precedence in cases where both // are set @@ -79,6 +93,9 @@ func DetermineEffectiveSecurityContext(pod *v1.Pod, container *v1.Container) *v1 if containerSc.WindowsOptions.RunAsUserName != nil { effectiveSc.WindowsOptions.RunAsUserName = containerSc.WindowsOptions.RunAsUserName } + if containerSc.WindowsOptions.HostProcess != nil { + effectiveSc.WindowsOptions.HostProcess = containerSc.WindowsOptions.HostProcess + } } if containerSc.Capabilities != nil {