From 3f5675880d29b203d577149a257fa5fa0f2b5397 Mon Sep 17 00:00:00 2001 From: Jean Rouge Date: Mon, 4 Feb 2019 16:42:52 -0800 Subject: [PATCH] Kubelet changes for Windows GMSA support This patch comprises the kubelet changes outlined in the GMSA KEP (https://github.com/kubernetes/enhancements/blob/master/keps/sig-windows/20181221-windows-group-managed-service-accounts-for-container-identity.md) to add GMSA support to Windows workloads. More precisely, it includes the logic proposed in the KEP to resolve which GMSA spec should be applied to which containers, and changes `dockershim` to copy the relevant GMSA credential specs to Windows registry values prior to creating the container, passing them down to docker itself, and finally removing the values from the registry afterwards; both these changes need to be activated with the `WindowsGMSA` feature gate. Includes unit tests. Signed-off-by: Jean Rouge --- pkg/features/kube_features.go | 6 + pkg/kubelet/dockershim/BUILD | 9 + pkg/kubelet/dockershim/docker_container.go | 9 + .../docker_container_unsupported.go | 39 +++ .../dockershim/docker_container_windows.go | 180 ++++++++++++++ .../docker_container_windows_test.go | 233 ++++++++++++++++++ pkg/kubelet/dockershim/naming.go | 1 + pkg/kubelet/kuberuntime/BUILD | 1 + .../kuberuntime_container_windows.go | 43 ++++ .../kuberuntime_container_windows_test.go | 82 ++++++ 10 files changed, 603 insertions(+) create mode 100644 pkg/kubelet/dockershim/docker_container_unsupported.go create mode 100644 pkg/kubelet/dockershim/docker_container_windows.go create mode 100644 pkg/kubelet/dockershim/docker_container_windows_test.go create mode 100644 pkg/kubelet/kuberuntime/kuberuntime_container_windows_test.go diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index a3b523a8d77..d6827246908 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -404,6 +404,12 @@ const ( // // Enables the AWS EBS in-tree driver to AWS EBS CSI Driver migration feature. CSIMigrationAWS utilfeature.Feature = "CSIMigrationAWS" + + // owner: @wk8 + // alpha: v1.14 + // + // Enables GMSA support for Windows workloads. + WindowsGMSA utilfeature.Feature = "WindowsGMSA" ) func init() { diff --git a/pkg/kubelet/dockershim/BUILD b/pkg/kubelet/dockershim/BUILD index 5915ed02124..4cc4e520794 100644 --- a/pkg/kubelet/dockershim/BUILD +++ b/pkg/kubelet/dockershim/BUILD @@ -7,6 +7,8 @@ go_library( "doc.go", "docker_checkpoint.go", "docker_container.go", + "docker_container_unsupported.go", + "docker_container_windows.go", "docker_image.go", "docker_image_linux.go", "docker_image_unsupported.go", @@ -71,8 +73,11 @@ go_library( "//vendor/k8s.io/utils/exec:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:windows": [ + "//pkg/features:go_default_library", "//pkg/kubelet/apis:go_default_library", "//pkg/kubelet/winstats:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/golang.org/x/sys/windows/registry:go_default_library", ], "//conditions:default": [], }), @@ -84,6 +89,7 @@ go_test( "convert_test.go", "docker_checkpoint_test.go", "docker_container_test.go", + "docker_container_windows_test.go", "docker_image_test.go", "docker_sandbox_test.go", "docker_service_test.go", @@ -118,6 +124,9 @@ go_test( "@io_bazel_rules_go//go/platform:linux": [ "//staging/src/k8s.io/api/core/v1:go_default_library", ], + "@io_bazel_rules_go//go/platform:windows": [ + "//vendor/golang.org/x/sys/windows/registry:go_default_library", + ], "//conditions:default": [], }), ) diff --git a/pkg/kubelet/dockershim/docker_container.go b/pkg/kubelet/dockershim/docker_container.go index 3c6b9b48497..47908d3fc40 100644 --- a/pkg/kubelet/dockershim/docker_container.go +++ b/pkg/kubelet/dockershim/docker_container.go @@ -162,11 +162,20 @@ func (ds *dockerService) CreateContainer(_ context.Context, r *runtimeapi.Create hc.SecurityOpt = append(hc.SecurityOpt, securityOpts...) + cleanupInfo, err := ds.applyPlatformSpecificDockerConfig(r, &createConfig) + if err != nil { + return nil, err + } + createResp, err := ds.client.CreateContainer(createConfig) if err != nil { createResp, err = recoverFromCreationConflictIfNeeded(ds.client, createConfig, err) } + if err = ds.performPlatformSpecificContainerCreationCleanup(cleanupInfo); err != nil { + return nil, err + } + if createResp != nil { return &runtimeapi.CreateContainerResponse{ContainerId: createResp.ID}, nil } diff --git a/pkg/kubelet/dockershim/docker_container_unsupported.go b/pkg/kubelet/dockershim/docker_container_unsupported.go new file mode 100644 index 00000000000..38e7c9fdbac --- /dev/null +++ b/pkg/kubelet/dockershim/docker_container_unsupported.go @@ -0,0 +1,39 @@ +// +build !windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockershim + +import ( + dockertypes "github.com/docker/docker/api/types" + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" +) + +type containerCreationCleanupInfo struct{} + +// applyPlatformSpecificDockerConfig applies platform-specific configurations to a dockertypes.ContainerCreateConfig struct. +// The containerCreationCleanupInfo struct it returns will be passed as is to performPlatformSpecificContainerCreationCleanup +// after the container has been created. +func (ds *dockerService) applyPlatformSpecificDockerConfig(*runtimeapi.CreateContainerRequest, *dockertypes.ContainerCreateConfig) (*containerCreationCleanupInfo, error) { + return nil, nil +} + +// performPlatformSpecificContainerCreationCleanup is responsible for doing any platform-specific cleanup +// after a container creation. +func (ds *dockerService) performPlatformSpecificContainerCreationCleanup(*containerCreationCleanupInfo) error { + return nil +} diff --git a/pkg/kubelet/dockershim/docker_container_windows.go b/pkg/kubelet/dockershim/docker_container_windows.go new file mode 100644 index 00000000000..e102edbbbbe --- /dev/null +++ b/pkg/kubelet/dockershim/docker_container_windows.go @@ -0,0 +1,180 @@ +// +build windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockershim + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + + "golang.org/x/sys/windows/registry" + + dockertypes "github.com/docker/docker/api/types" + dockercontainer "github.com/docker/docker/api/types/container" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" + kubefeatures "k8s.io/kubernetes/pkg/features" + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" + "k8s.io/kubernetes/pkg/kubelet/kuberuntime" +) + +type containerCreationCleanupInfo struct { + gMSARegistryValueName string +} + +// applyPlatformSpecificDockerConfig applies platform-specific configurations to a dockertypes.ContainerCreateConfig struct. +// The containerCreationCleanupInfo struct it returns will be passed as is to performPlatformSpecificContainerCreationCleanup +// after the container has been created. +func (ds *dockerService) applyPlatformSpecificDockerConfig(request *runtimeapi.CreateContainerRequest, createConfig *dockertypes.ContainerCreateConfig) (*containerCreationCleanupInfo, error) { + cleanupInfo := &containerCreationCleanupInfo{} + + if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsGMSA) { + if err := applyGMSAConfig(request, createConfig, cleanupInfo); err != nil { + return nil, err + } + } + + return cleanupInfo, nil +} + +// applyGMSAConfig looks at the kuberuntime.GMSASpecContainerAnnotationKey container annotation; if present, +// it copies its contents to a unique registry value, and sets a SecurityOpt on the config pointing to that registry value. +// We use registry values instead of files since their location cannot change - as opposed to credential spec files, +// whose location could potentially change down the line, or even be unknown (eg if docker is not installed on the +// C: drive) +// When docker supports passing a credential spec's contents directly, we should switch to using that +// as it will avoid cluttering the registry. +func applyGMSAConfig(request *runtimeapi.CreateContainerRequest, createConfig *dockertypes.ContainerCreateConfig, cleanupInfo *containerCreationCleanupInfo) error { + config := request.GetConfig() + credSpec := config.Annotations[kuberuntime.GMSASpecContainerAnnotationKey] + if credSpec == "" { + return nil + } + + valueName, err := copyGMSACredSpecToRegistryValue(credSpec, makeContainerName(request.GetSandboxConfig(), config)) + if err != nil { + return err + } + + if createConfig.HostConfig == nil { + createConfig.HostConfig = &dockercontainer.HostConfig{} + } + + createConfig.HostConfig.SecurityOpt = append(createConfig.HostConfig.SecurityOpt, "credentialspec=registry://"+valueName) + cleanupInfo.gMSARegistryValueName = valueName + + return nil +} + +const ( + registryNamePrefix = "k8s-cred-spec-" + // same as https://github.com/moby/moby/blob/93d994e29c9cc8d81f1b0477e28d705fa7e2cd72/daemon/oci_windows.go#L23 + credentialSpecRegistryLocation = `SOFTWARE\Microsoft\Windows NT\CurrentVersion\Virtualization\Containers\CredentialSpecs` +) + +// useful to allow mocking the registry in tests +type registryKey interface { + SetStringValue(name, value string) error + DeleteValue(name string) error + Close() error +} + +var registryCreateKeyFunc = func(baseKey registry.Key, path string, access uint32) (registryKey, bool, error) { + return registry.CreateKey(baseKey, path, access) +} + +// and same for random +var randomReader = rand.Reader + +// copyGMSACredSpecToRegistryKey copies the credential specs to a unique registry value, and returns its name. +// To avoid leaking registry keys over the life of the node, we generate a unique name for that value, and clean +// it up after creating the container. +func copyGMSACredSpecToRegistryValue(credSpec string, dockerContainerName string) (string, error) { + valueName, err := gMSARegistryValueName(credSpec, dockerContainerName) + if err != nil { + return "", err + } + + // write to the registry + key, _, err := registryCreateKeyFunc(registry.LOCAL_MACHINE, credentialSpecRegistryLocation, registry.SET_VALUE) + if err != nil { + return "", fmt.Errorf("unable to open registry key %s: %v", credentialSpecRegistryLocation, err) + } + defer key.Close() + if err = key.SetStringValue(valueName, credSpec); err != nil { + return "", fmt.Errorf("unable to write into registry value %s/%s: %v", credentialSpecRegistryLocation, valueName, err) + } + + return valueName, nil +} + +// gMSARegistryValueName computes the name of the registry value where to store the GMSA cred spec contents. +// The value's name is computed by concatenating the docker container's name (guaranteed to be unique over the +// container's lifetime), the value itself, and an additional 64 random bytes. +func gMSARegistryValueName(inputs ...string) (string, error) { + hasher := sha256.New() + for _, s := range inputs { + // according to the doc, that can never return an error + io.WriteString(hasher, s) + } + randBytes := make([]byte, 64) + if _, err := randomReader.Read(randBytes); err != nil { + return "", fmt.Errorf("unable to generate random string: %v", err) + } + hasher.Write(randBytes) + + return registryNamePrefix + hex.EncodeToString(hasher.Sum(nil)), nil +} + +// performPlatformSpecificContainerCreationCleanup is responsible for doing any platform-specific cleanup +// after a container creation. +func (ds *dockerService) performPlatformSpecificContainerCreationCleanup(cleanupInfo *containerCreationCleanupInfo) error { + if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsGMSA) { + // this is best effort, we don't bubble errors upstream as failing to remove the GMSA registry keys shouldn't + // prevent k8s from working correctly, and the leaked registry keys are not a major concern anyway: + // they don't contain any secret, and they're sufficiently random to prevent collisions with + // future ones + if err := removeGMSARegistryValue(cleanupInfo); err != nil { + klog.Warningf("won't remove GMSA cred spec registry value: %v", err) + } + } + + return nil +} + +// removeGMSARegistryValue removes the registry value containing the GMSA cred spec for this container, if any. +func removeGMSARegistryValue(cleanupInfo *containerCreationCleanupInfo) error { + if cleanupInfo.gMSARegistryValueName == "" { + return nil + } + + key, _, err := registryCreateKeyFunc(registry.LOCAL_MACHINE, credentialSpecRegistryLocation, registry.SET_VALUE) + if err != nil { + return fmt.Errorf("unable to open registry key %s: %v", credentialSpecRegistryLocation, err) + } + defer key.Close() + if err = key.DeleteValue(cleanupInfo.gMSARegistryValueName); err != nil { + return fmt.Errorf("unable to remove registry value %s/%s: %v", credentialSpecRegistryLocation, cleanupInfo.gMSARegistryValueName, err) + } + + return nil +} diff --git a/pkg/kubelet/dockershim/docker_container_windows_test.go b/pkg/kubelet/dockershim/docker_container_windows_test.go new file mode 100644 index 00000000000..e4f553a7076 --- /dev/null +++ b/pkg/kubelet/dockershim/docker_container_windows_test.go @@ -0,0 +1,233 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockershim + +import ( + "bytes" + "fmt" + "regexp" + "strings" + "testing" + + dockertypes "github.com/docker/docker/api/types" + "github.com/stretchr/testify/assert" + "golang.org/x/sys/windows/registry" + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" +) + +type dummyRegistryKey struct { + setStringError error + stringValues [][]string + deleteValueError error + deletedValueNames []string + closed bool +} + +func (k *dummyRegistryKey) SetStringValue(name, value string) error { + k.stringValues = append(k.stringValues, []string{name, value}) + return k.setStringError +} + +func (k *dummyRegistryKey) DeleteValue(name string) error { + k.deletedValueNames = append(k.deletedValueNames, name) + return k.deleteValueError +} + +func (k *dummyRegistryKey) Close() error { + k.closed = true + return nil +} + +func TestApplyGMSAConfig(t *testing.T) { + dummyCredSpec := "test cred spec contents" + randomBytes := []byte{85, 205, 157, 137, 41, 50, 187, 175, 242, 115, 92, 212, 181, 70, 56, 20, 172, 17, 100, 178, 19, 42, 217, 177, 240, 37, 127, 123, 53, 250, 61, 157, 11, 41, 69, 160, 117, 163, 51, 118, 53, 86, 167, 111, 137, 78, 195, 229, 50, 144, 178, 209, 66, 107, 144, 165, 184, 92, 10, 17, 229, 163, 194, 12} + expectedHash := "8975ef53024af213c1aca6dfc6e2e48f42c3a984a79e67b140627b8d96007c2a" + expectedValueName := "k8s-cred-spec-" + expectedHash + + sandboxConfig := &runtimeapi.PodSandboxConfig{ + Metadata: &runtimeapi.PodSandboxMetadata{ + Namespace: "namespace", + Uid: "uid", + }, + } + + containerMeta := &runtimeapi.ContainerMetadata{ + Name: "container_name", + Attempt: 12, + } + + requestWithoutGMSAAnnotation := &runtimeapi.CreateContainerRequest{ + Config: &runtimeapi.ContainerConfig{Metadata: containerMeta}, + SandboxConfig: sandboxConfig, + } + + requestWithGMSAAnnotation := &runtimeapi.CreateContainerRequest{ + Config: &runtimeapi.ContainerConfig{ + Metadata: containerMeta, + Annotations: map[string]string{"container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec}, + }, + SandboxConfig: sandboxConfig, + } + + t.Run("happy path", func(t *testing.T) { + key := &dummyRegistryKey{} + defer setRegistryCreateKeyFunc(t, key)() + defer setRandomReader(randomBytes)() + + createConfig := &dockertypes.ContainerCreateConfig{} + cleanupInfo := &containerCreationCleanupInfo{} + err := applyGMSAConfig(requestWithGMSAAnnotation, createConfig, cleanupInfo) + + assert.Nil(t, err) + + // the registry key should have been properly created + assert.Equal(t, 1, len(key.stringValues)) + assert.Equal(t, []string{expectedValueName, dummyCredSpec}, key.stringValues[0]) + assert.True(t, key.closed) + + // the create config's security opt should have been populated + assert.Equal(t, createConfig.HostConfig.SecurityOpt, []string{"credentialspec=registry://" + expectedValueName}) + + // and the name of that value should have been saved to the cleanup info + assert.Equal(t, expectedValueName, cleanupInfo.gMSARegistryValueName) + }) + t.Run("happy path with a truly random string", func(t *testing.T) { + defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{})() + + createConfig := &dockertypes.ContainerCreateConfig{} + cleanupInfo := &containerCreationCleanupInfo{} + err := applyGMSAConfig(requestWithGMSAAnnotation, createConfig, cleanupInfo) + + assert.Nil(t, err) + + secOpt := createConfig.HostConfig.SecurityOpt[0] + + expectedPrefix := "credentialspec=registry://k8s-cred-spec-" + assert.Equal(t, expectedPrefix, secOpt[:len(expectedPrefix)]) + + hash := secOpt[len(expectedPrefix):] + hexRegex, _ := regexp.Compile("^[0-9a-f]{64}$") + assert.True(t, hexRegex.MatchString(hash)) + assert.NotEqual(t, expectedHash, hash) + + assert.Equal(t, "k8s-cred-spec-"+hash, cleanupInfo.gMSARegistryValueName) + }) + t.Run("if there's an error opening the registry key", func(t *testing.T) { + defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{}, fmt.Errorf("dummy error"))() + + err := applyGMSAConfig(requestWithGMSAAnnotation, &dockertypes.ContainerCreateConfig{}, &containerCreationCleanupInfo{}) + + assert.NotNil(t, err) + assert.True(t, strings.Contains(err.Error(), "unable to open registry key")) + }) + t.Run("if there's an error writing the registry key", func(t *testing.T) { + key := &dummyRegistryKey{} + key.setStringError = fmt.Errorf("dummy error") + defer setRegistryCreateKeyFunc(t, key)() + + err := applyGMSAConfig(requestWithGMSAAnnotation, &dockertypes.ContainerCreateConfig{}, &containerCreationCleanupInfo{}) + + assert.NotNil(t, err) + assert.True(t, strings.Contains(err.Error(), "unable to write into registry value")) + assert.True(t, key.closed) + }) + t.Run("if there is no GMSA annotation", func(t *testing.T) { + createConfig := &dockertypes.ContainerCreateConfig{} + + err := applyGMSAConfig(requestWithoutGMSAAnnotation, createConfig, &containerCreationCleanupInfo{}) + + assert.Nil(t, err) + assert.Nil(t, createConfig.HostConfig) + }) +} + +func TestRemoveGMSARegistryValue(t *testing.T) { + emptyCleanupInfo := &containerCreationCleanupInfo{} + + valueName := "k8s-cred-spec-8975ef53024af213c1aca6dfc6e2e48f42c3a984a79e67b140627b8d96007c2a" + cleanupInfoWithValue := &containerCreationCleanupInfo{gMSARegistryValueName: valueName} + + t.Run("it does remove the registry value", func(t *testing.T) { + key := &dummyRegistryKey{} + defer setRegistryCreateKeyFunc(t, key)() + + err := removeGMSARegistryValue(cleanupInfoWithValue) + + assert.Nil(t, err) + + // the registry key should have been properly deleted + assert.Equal(t, 1, len(key.deletedValueNames)) + assert.Equal(t, []string{valueName}, key.deletedValueNames) + assert.True(t, key.closed) + }) + t.Run("if there's an error opening the registry key", func(t *testing.T) { + defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{}, fmt.Errorf("dummy error"))() + + err := removeGMSARegistryValue(cleanupInfoWithValue) + + assert.NotNil(t, err) + assert.True(t, strings.Contains(err.Error(), "unable to open registry key")) + }) + t.Run("if there's an error writing the registry key", func(t *testing.T) { + key := &dummyRegistryKey{} + key.deleteValueError = fmt.Errorf("dummy error") + defer setRegistryCreateKeyFunc(t, key)() + + err := removeGMSARegistryValue(cleanupInfoWithValue) + + assert.NotNil(t, err) + assert.True(t, strings.Contains(err.Error(), "unable to remove registry value")) + assert.True(t, key.closed) + }) + t.Run("if there's no registry value to be removed", func(t *testing.T) { + err := removeGMSARegistryValue(emptyCleanupInfo) + + assert.Nil(t, err) + }) +} + +// setRegistryCreateKeyFunc replaces the registryCreateKeyFunc package variable, and returns a function +// to be called to revert the change when done with testing. +func setRegistryCreateKeyFunc(t *testing.T, key *dummyRegistryKey, err ...error) func() { + previousRegistryCreateKeyFunc := registryCreateKeyFunc + + registryCreateKeyFunc = func(baseKey registry.Key, path string, access uint32) (registryKey, bool, error) { + // this should always be called with exactly the same arguments + assert.Equal(t, registry.LOCAL_MACHINE, baseKey) + assert.Equal(t, credentialSpecRegistryLocation, path) + assert.Equal(t, uint32(registry.SET_VALUE), access) + + if len(err) > 0 { + return nil, false, err[0] + } + return key, false, nil + } + + return func() { + registryCreateKeyFunc = previousRegistryCreateKeyFunc + } +} + +// setRandomReader replaces the randomReader package variable with a dummy reader that returns the provided +// byte slice, and returns a function to be called to revert the change when done with testing. +func setRandomReader(b []byte) func() { + previousRandomReader := randomReader + randomReader = bytes.NewReader(b) + return func() { + randomReader = previousRandomReader + } +} diff --git a/pkg/kubelet/dockershim/naming.go b/pkg/kubelet/dockershim/naming.go index a2a97c2da4e..cb2bc467b1b 100644 --- a/pkg/kubelet/dockershim/naming.go +++ b/pkg/kubelet/dockershim/naming.go @@ -66,6 +66,7 @@ func makeSandboxName(s *runtimeapi.PodSandboxConfig) string { }, nameDelimiter) } +// makeContainerName generates a container name that's guaranteed to be unique on its host. func makeContainerName(s *runtimeapi.PodSandboxConfig, c *runtimeapi.ContainerConfig) string { return strings.Join([]string{ kubePrefix, // 0 diff --git a/pkg/kubelet/kuberuntime/BUILD b/pkg/kubelet/kuberuntime/BUILD index 12c16e7e548..c7eec262375 100644 --- a/pkg/kubelet/kuberuntime/BUILD +++ b/pkg/kubelet/kuberuntime/BUILD @@ -89,6 +89,7 @@ go_test( "instrumented_services_test.go", "kuberuntime_container_linux_test.go", "kuberuntime_container_test.go", + "kuberuntime_container_windows_test.go", "kuberuntime_gc_test.go", "kuberuntime_image_test.go", "kuberuntime_manager_test.go", diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go b/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go index 77165e81b21..da606c3cfbf 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go @@ -23,6 +23,8 @@ import ( "github.com/docker/docker/pkg/sysinfo" "k8s.io/api/core/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + kubefeatures "k8s.io/kubernetes/pkg/features" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "k8s.io/kubernetes/pkg/securitycontext" @@ -35,6 +37,10 @@ func (m *kubeGenericRuntimeManager) applyPlatformSpecificContainerConfig(config return err } + if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsGMSA) { + determineEffectiveSecurityContext(config, container, pod) + } + config.Windows = windowsConfig return nil } @@ -97,3 +103,40 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1 return wc, nil } + +const ( + // GMSASpecContainerAnnotationKey is the container annotation where we store the contents of the GMSA credential spec to use. + GMSASpecContainerAnnotationKey = "container.alpha.windows.kubernetes.io/gmsa-credential-spec" + // gMSAContainerSpecPodAnnotationKeySuffix is the suffix of the pod annotation where the GMSA webhook admission controller + // stores the contents of the GMSA credential spec for a given container (the full annotation being the container's name + // with this suffix appended). + gMSAContainerSpecPodAnnotationKeySuffix = "." + GMSASpecContainerAnnotationKey + // gMSAPodSpecPodAnnotationKey is the pod annotation where the GMSA webhook admission controller stores the contents of the GMSA + // credential spec to use for containers that do not have their own specific GMSA cred spec set via a + // gMSAContainerSpecPodAnnotationKeySuffix annotation as explained above + gMSAPodSpecPodAnnotationKey = "pod.alpha.windows.kubernetes.io/gmsa-credential-spec" +) + +// determineEffectiveSecurityContext determines the effective GMSA credential spec and, if any, copies it to the container's +// GMSASpecContainerAnnotationKey annotation. +func determineEffectiveSecurityContext(config *runtimeapi.ContainerConfig, container *v1.Container, pod *v1.Pod) { + var containerCredSpec string + + containerGMSAPodAnnotation := container.Name + gMSAContainerSpecPodAnnotationKeySuffix + if pod.Annotations[containerGMSAPodAnnotation] != "" { + containerCredSpec = pod.Annotations[containerGMSAPodAnnotation] + } else if pod.Annotations[gMSAPodSpecPodAnnotationKey] != "" { + containerCredSpec = pod.Annotations[gMSAPodSpecPodAnnotationKey] + } + + if containerCredSpec != "" { + if config.Annotations == nil { + config.Annotations = make(map[string]string) + } + config.Annotations[GMSASpecContainerAnnotationKey] = containerCredSpec + } else { + // the annotation shouldn't be present, but let's err on the side of caution: + // it should only be set here and nowhere else + delete(config.Annotations, GMSASpecContainerAnnotationKey) + } +} diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_windows_test.go b/pkg/kubelet/kuberuntime/kuberuntime_container_windows_test.go new file mode 100644 index 00000000000..e10b43eca9b --- /dev/null +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_windows_test.go @@ -0,0 +1,82 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kuberuntime + +import ( + "github.com/stretchr/testify/assert" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" +) + +func TestDetermineEffectiveSecurityContext(t *testing.T) { + containerName := "container_name" + container := &corev1.Container{Name: containerName} + dummyCredSpec := "test cred spec contents" + + buildPod := func(annotations map[string]string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, + } + } + + t.Run("when there's a specific GMSA for that pod, and no pod-wide GMSA", func(t *testing.T) { + containerConfig := &runtimeapi.ContainerConfig{} + + pod := buildPod(map[string]string{ + "container_name.container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec, + }) + + determineEffectiveSecurityContext(containerConfig, container, pod) + + assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"]) + }) + t.Run("when there's a specific GMSA for that pod, and a pod-wide GMSA", func(t *testing.T) { + containerConfig := &runtimeapi.ContainerConfig{} + + pod := buildPod(map[string]string{ + "container_name.container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec, + "pod.alpha.windows.kubernetes.io/gmsa-credential-spec": "should be ignored", + }) + + determineEffectiveSecurityContext(containerConfig, container, pod) + + assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"]) + }) + t.Run("when there's no specific GMSA for that pod, and a pod-wide GMSA", func(t *testing.T) { + containerConfig := &runtimeapi.ContainerConfig{} + + pod := buildPod(map[string]string{ + "pod.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec, + }) + + determineEffectiveSecurityContext(containerConfig, container, pod) + + assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"]) + }) + t.Run("when there's no specific GMSA for that pod, and no pod-wide GMSA", func(t *testing.T) { + containerConfig := &runtimeapi.ContainerConfig{} + + determineEffectiveSecurityContext(containerConfig, container, &corev1.Pod{}) + + assert.Nil(t, containerConfig.Annotations) + }) +}