Kubelet updates for Windows HostProcess Containers

This commit is contained in:
marosset 2021-02-28 23:25:41 -08:00 committed by James Sturtevant
parent 51a02fdb80
commit fd94032b21
6 changed files with 340 additions and 4 deletions

View File

@ -31,6 +31,7 @@ import (
"k8s.io/client-go/tools/record"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
sc "k8s.io/kubernetes/pkg/securitycontext"
hashutil "k8s.io/kubernetes/pkg/util/hash"
"k8s.io/kubernetes/third_party/forked/golang/expansion"
utilsnet "k8s.io/utils/net"
@ -310,6 +311,34 @@ func HasPrivilegedContainer(pod *v1.Pod) bool {
return hasPrivileged
}
// HasWindowsHostProcessContainer returns true if any of the containers in a pod are HostProcess containers.
func HasWindowsHostProcessContainer(pod *v1.Pod) bool {
var hasHostProcess bool
podutil.VisitContainers(&pod.Spec, podutil.AllFeatureEnabledContainers(), func(c *v1.Container, containerType podutil.ContainerType) bool {
if sc.HasWindowsHostProcessRequest(pod, c) {
hasHostProcess = true
return false
}
return true
})
return hasHostProcess
}
// AllContainersAreWindowsHostProcess returns true if all containres in a pod are HostProcess containers.
func AllContainersAreWindowsHostProcess(pod *v1.Pod) bool {
allHostProcess := true
podutil.VisitContainers(&pod.Spec, podutil.AllFeatureEnabledContainers(), func(c *v1.Container, containerType podutil.ContainerType) bool {
if !sc.HasWindowsHostProcessRequest(pod, c) {
allHostProcess = false
return false
}
return true
})
return allHostProcess
}
// MakePortMappings creates internal port mapping from api port mapping.
func MakePortMappings(container *v1.Container) (ports []PortMapping) {
names := make(map[string]struct{})

View File

@ -19,14 +19,16 @@ limitations under the License.
package kuberuntime
import (
"fmt"
"runtime"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/features"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/securitycontext"
"k8s.io/klog/v2"
)
// applyPlatformSpecificContainerConfig applies platform specific configurations to runtimeapi.ContainerConfig.
@ -122,5 +124,12 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1
wc.SecurityContext.RunAsUsername = *effectiveSc.WindowsOptions.RunAsUserName
}
if securitycontext.HasWindowsHostProcessRequest(pod, container) {
if !utilfeature.DefaultFeatureGate.Enabled(features.WindowsHostProcessContainers) {
return nil, fmt.Errorf("pod contains HostProcess containers but feature 'WindowsHostProcessContainers' is not enabled")
}
wc.SecurityContext.HostProcess = true
}
return wc, nil
}

View File

@ -25,8 +25,10 @@ import (
v1 "k8s.io/api/core/v1"
kubetypes "k8s.io/apimachinery/pkg/types"
utilfeature "k8s.io/apiserver/pkg/util/feature"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/features"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/types"
"k8s.io/kubernetes/pkg/kubelet/util"
@ -138,6 +140,14 @@ func (m *kubeGenericRuntimeManager) generatePodSandboxConfig(pod *v1.Pod, attemp
}
podSandboxConfig.Linux = lc
if runtime.GOOS == "windows" {
wc, err := m.generatePodSandboxWindowsConfig(pod)
if err != nil {
return nil, err
}
podSandboxConfig.Windows = wc
}
return podSandboxConfig, nil
}
@ -206,6 +216,54 @@ func (m *kubeGenericRuntimeManager) generatePodSandboxLinuxConfig(pod *v1.Pod) (
return lc, nil
}
// generatePodSandboxWindowsConfig generates WindowsPodSandboxConfig from v1.Pod.
// On Windows this will get called in addition to LinuxPodSandboxConfig because not all relevant fields have been added to
// WindowsPodSandboxConfig at this time.
func (m *kubeGenericRuntimeManager) generatePodSandboxWindowsConfig(pod *v1.Pod) (*runtimeapi.WindowsPodSandboxConfig, error) {
wc := &runtimeapi.WindowsPodSandboxConfig{
SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{},
}
sc := pod.Spec.SecurityContext
if sc == nil || sc.WindowsOptions == nil {
return wc, nil
}
wo := sc.WindowsOptions
if wo.GMSACredentialSpec != nil {
wc.SecurityContext.CredentialSpec = *wo.GMSACredentialSpec
}
if wo.RunAsUserName != nil {
wc.SecurityContext.RunAsUsername = *wo.RunAsUserName
}
if kubecontainer.HasWindowsHostProcessContainer(pod) {
// Pods containing HostProcess containers should fail to schedule if feature is not
// enabled instead of trying to schedule containers as regular containers as stated in
// PRR review.
if !utilfeature.DefaultFeatureGate.Enabled(features.WindowsHostProcessContainers) {
return nil, fmt.Errorf("pod contains HostProcess containers but feature 'WindowsHostProcessContainers' is not enabled")
}
if wo.HostProcess != nil && !*wo.HostProcess {
return nil, fmt.Errorf("pod must not contain any HostProcess containers if Pod's WindowsOptions.HostProcess is set to false")
}
// At present Windows all containers in a Windows pod must be HostProcess containers
// and HostNetwork is required to be set.
if !kubecontainer.AllContainersAreWindowsHostProcess(pod) {
return nil, fmt.Errorf("pod must not contain both HostProcess and non-HostProcess containers")
}
if !kubecontainer.IsHostNetworkPod(pod) {
return nil, fmt.Errorf("hostNetwork is required if Pod contains HostProcess containers")
}
wc.SecurityContext.HostProcess = true
}
return wc, nil
}
// getKubeletSandboxes lists all (or just the running) sandboxes managed by kubelet.
func (m *kubeGenericRuntimeManager) getKubeletSandboxes(all bool) ([]*runtimeapi.PodSandbox, error) {
var filter *runtimeapi.PodSandboxFilter

View File

@ -17,6 +17,7 @@ limitations under the License.
package kuberuntime
import (
"fmt"
"os"
"path/filepath"
"testing"
@ -25,7 +26,10 @@ import (
"github.com/stretchr/testify/require"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
"k8s.io/kubernetes/pkg/features"
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
"k8s.io/kubernetes/pkg/kubelet/runtimeclass"
rctest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
@ -172,3 +176,187 @@ func newSeccompPod(podFieldProfile, containerFieldProfile *v1.SeccompProfile, po
}
return pod
}
func TestGeneratePodSandboxWindowsConfig(t *testing.T) {
_, _, m, err := createTestRuntimeManager()
require.NoError(t, err)
const containerName = "container"
gmsaCreds := "gmsa-creds"
userName := "SYSTEM"
trueVar := true
falseVar := false
testCases := []struct {
name string
hostProcessFeatureEnabled bool
podSpec *v1.PodSpec
expectedWindowsConfig *runtimeapi.WindowsPodSandboxConfig
expectedError error
}{
{
name: "Empty PodSecurityContext",
hostProcessFeatureEnabled: false,
podSpec: &v1.PodSpec{
Containers: []v1.Container{{
Name: containerName,
}},
},
expectedWindowsConfig: &runtimeapi.WindowsPodSandboxConfig{
SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{},
},
expectedError: nil,
},
{
name: "GMSACredentialSpec in PodSecurityContext",
hostProcessFeatureEnabled: false,
podSpec: &v1.PodSpec{
SecurityContext: &v1.PodSecurityContext{
WindowsOptions: &v1.WindowsSecurityContextOptions{
GMSACredentialSpec: &gmsaCreds,
},
},
Containers: []v1.Container{{
Name: containerName,
}},
},
expectedWindowsConfig: &runtimeapi.WindowsPodSandboxConfig{
SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{
CredentialSpec: "gmsa-creds",
},
},
expectedError: nil,
},
{
name: "RunAsUserName in PodSecurityContext",
hostProcessFeatureEnabled: false,
podSpec: &v1.PodSpec{
SecurityContext: &v1.PodSecurityContext{
WindowsOptions: &v1.WindowsSecurityContextOptions{
RunAsUserName: &userName,
},
},
Containers: []v1.Container{{
Name: containerName,
}},
},
expectedWindowsConfig: &runtimeapi.WindowsPodSandboxConfig{
SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{
RunAsUsername: "SYSTEM",
},
},
expectedError: nil,
},
{
name: "Pod with HostProcess containers and feature gate disabled",
hostProcessFeatureEnabled: false,
podSpec: &v1.PodSpec{
SecurityContext: &v1.PodSecurityContext{
WindowsOptions: &v1.WindowsSecurityContextOptions{
HostProcess: &trueVar,
},
},
Containers: []v1.Container{{
Name: containerName,
}},
},
expectedWindowsConfig: nil,
expectedError: fmt.Errorf("pod contains HostProcess containers but feature 'WindowsHostProcessContainers' is not enabled"),
},
{
name: "Pod with HostProcess containers and non-HostProcess containers",
hostProcessFeatureEnabled: true,
podSpec: &v1.PodSpec{
SecurityContext: &v1.PodSecurityContext{
WindowsOptions: &v1.WindowsSecurityContextOptions{
HostProcess: &trueVar,
},
},
Containers: []v1.Container{{
Name: containerName,
}, {
Name: containerName,
SecurityContext: &v1.SecurityContext{
WindowsOptions: &v1.WindowsSecurityContextOptions{
HostProcess: &falseVar,
},
},
}},
},
expectedWindowsConfig: nil,
expectedError: fmt.Errorf("pod must not contain both HostProcess and non-HostProcess containers"),
},
{
name: "Pod with HostProcess containers and HostNetwork not set",
hostProcessFeatureEnabled: true,
podSpec: &v1.PodSpec{
SecurityContext: &v1.PodSecurityContext{
WindowsOptions: &v1.WindowsSecurityContextOptions{
HostProcess: &trueVar,
},
},
Containers: []v1.Container{{
Name: containerName,
}},
},
expectedWindowsConfig: nil,
expectedError: fmt.Errorf("hostNetwork is required if Pod contains HostProcess containers"),
},
{
name: "Pod with HostProcess containers and HostNetwork set",
hostProcessFeatureEnabled: true,
podSpec: &v1.PodSpec{
HostNetwork: true,
SecurityContext: &v1.PodSecurityContext{
WindowsOptions: &v1.WindowsSecurityContextOptions{
HostProcess: &trueVar,
},
},
Containers: []v1.Container{{
Name: containerName,
}},
},
expectedWindowsConfig: &runtimeapi.WindowsPodSandboxConfig{
SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{
HostProcess: true,
},
},
expectedError: nil,
},
{
name: "Pod's WindowsOptions.HostProcess set to false and pod has HostProcess containers",
hostProcessFeatureEnabled: true,
podSpec: &v1.PodSpec{
HostNetwork: true,
SecurityContext: &v1.PodSecurityContext{
WindowsOptions: &v1.WindowsSecurityContextOptions{
HostProcess: &falseVar,
},
},
Containers: []v1.Container{{
Name: containerName,
SecurityContext: &v1.SecurityContext{
WindowsOptions: &v1.WindowsSecurityContextOptions{
HostProcess: &trueVar,
},
},
}},
},
expectedWindowsConfig: nil,
expectedError: fmt.Errorf("pod must not contain any HostProcess containers if Pod's WindowsOptions.HostProcess is set to false"),
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WindowsHostProcessContainers, testCase.hostProcessFeatureEnabled)()
pod := &v1.Pod{}
pod.Spec = *testCase.podSpec
wc, err := m.generatePodSandboxWindowsConfig(pod)
assert.Equal(t, wc, testCase.expectedWindowsConfig)
assert.Equal(t, err, testCase.expectedError)
})
}
}

View File

@ -18,13 +18,17 @@ package kuberuntime
import (
"encoding/json"
"runtime"
"strconv"
v1 "k8s.io/api/core/v1"
kubetypes "k8s.io/apimachinery/pkg/types"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/features"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/types"
sc "k8s.io/kubernetes/pkg/securitycontext"
)
const (
@ -38,6 +42,12 @@ const (
containerTerminationMessagePolicyLabel = "io.kubernetes.container.terminationMessagePolicy"
containerPreStopHandlerLabel = "io.kubernetes.container.preStopHandler"
containerPortsLabel = "io.kubernetes.container.ports"
// TODO: remove this annotation when moving to beta for Windows hostprocess containers
// xref: https://github.com/kubernetes/kubernetes/pull/99576/commits/42fb66073214eed6fe43fa8b1586f396e30e73e3#r635392090
// Currently, ContainerD on Windows does not yet fully support HostProcess containers
// but will pass annotations to hcsshim which does have support.
windowsHostProcessContainer = "microsoft.com/hostprocess-container"
)
type labeledPodSandboxInfo struct {
@ -89,7 +99,23 @@ func newPodLabels(pod *v1.Pod) map[string]string {
// newPodAnnotations creates pod annotations from v1.Pod.
func newPodAnnotations(pod *v1.Pod) map[string]string {
return pod.Annotations
annotations := map[string]string{}
// Get annotations from v1.Pod
for k, v := range pod.Annotations {
annotations[k] = v
}
if runtime.GOOS == "windows" && utilfeature.DefaultFeatureGate.Enabled(features.WindowsHostProcessContainers) {
if kubecontainer.HasWindowsHostProcessContainer(pod) {
// While WindowsHostProcessContainers is in alpha pass 'microsoft.com/hostprocess-container' annotation
// to pod sandbox creations request. ContainerD on Windows does not yet fully support HostProcess
// containers but will pass annotations to hcsshim which does have support.
annotations[windowsHostProcessContainer] = "true"
}
}
return annotations
}
// newContainerLabels creates container labels from v1.Container and v1.Pod.
@ -143,6 +169,15 @@ func newContainerAnnotations(container *v1.Container, pod *v1.Pod, restartCount
}
}
if runtime.GOOS == "windows" && utilfeature.DefaultFeatureGate.Enabled(features.WindowsHostProcessContainers) {
if sc.HasWindowsHostProcessRequest(pod, container) {
// While WindowsHostProcessContainers is in alpha pass 'microsoft.com/hostprocess-container' annotation
// to create containers request. ContainerD on Windows does not yet fully support HostProcess containers
// but will pass annotations to hcsshim which does have support.
annotations[windowsHostProcessContainer] = "true"
}
}
return annotations
}

View File

@ -44,6 +44,20 @@ func HasCapabilitiesRequest(container *v1.Container) bool {
return len(container.SecurityContext.Capabilities.Add) > 0 || len(container.SecurityContext.Capabilities.Drop) > 0
}
// HasWindowsHostProcessRequest returns true if container should run as HostProcess container,
// taking into account nils
func HasWindowsHostProcessRequest(pod *v1.Pod, container *v1.Container) bool {
effectiveSc := DetermineEffectiveSecurityContext(pod, container)
if effectiveSc.WindowsOptions == nil {
return false
}
if effectiveSc.WindowsOptions.HostProcess == nil {
return false
}
return *effectiveSc.WindowsOptions.HostProcess
}
// DetermineEffectiveSecurityContext returns a synthesized SecurityContext for reading effective configurations
// from the provided pod's and container's security context. Container's fields take precedence in cases where both
// are set
@ -79,6 +93,9 @@ func DetermineEffectiveSecurityContext(pod *v1.Pod, container *v1.Container) *v1
if containerSc.WindowsOptions.RunAsUserName != nil {
effectiveSc.WindowsOptions.RunAsUserName = containerSc.WindowsOptions.RunAsUserName
}
if containerSc.WindowsOptions.HostProcess != nil {
effectiveSc.WindowsOptions.HostProcess = containerSc.WindowsOptions.HostProcess
}
}
if containerSc.Capabilities != nil {