Review comments

* value names are now purely random
* cleaning up leaked registry keys at Kubelet init
* fixing a small bug masking create errors

Signed-off-by: Jean Rouge <rougej+github@gmail.com>

pkg/kubelet/dockershim/docker_container.go

@@ -167,19 +167,23 @@ func (ds *dockerService) CreateContainer(_ context.Context, r *runtimeapi.Create
 		return nil, err
 	}
 
-	createResp, err := ds.client.CreateContainer(createConfig)
+	createResp, createErr := ds.client.CreateContainer(createConfig)
-	if err != nil {
+	if createErr != nil {
-		createResp, err = recoverFromCreationConflictIfNeeded(ds.client, createConfig, err)
+		createResp, createErr = recoverFromCreationConflictIfNeeded(ds.client, createConfig, createErr)
 	}
 
 	if err = ds.performPlatformSpecificContainerCreationCleanup(cleanupInfo); err != nil {
+		if createErr != nil {
+			// that one is more important to surface
+			return nil, createErr
+		}
 		return nil, err
 	}
 
 	if createResp != nil {
 		return &runtimeapi.CreateContainerResponse{ContainerId: createResp.ID}, nil
 	}
-	return nil, err
+	return nil, createErr
 }
 
 // getContainerLogPath returns the container log path specified by kubelet and the real

pkg/kubelet/dockershim/docker_container_unsupported.go

@@ -37,3 +37,10 @@ func (ds *dockerService) applyPlatformSpecificDockerConfig(*runtimeapi.CreateCon
 func (ds *dockerService) performPlatformSpecificContainerCreationCleanup(*containerCreationCleanupInfo) error {
 	return nil
 }
+
+// platformSpecificContainerCreationKubeletInitCleanup is called when the kubelet
+// is starting, and is meant to clean up any cruft left by previous runs;
+// errors are simply logged, but don't prevent the kubelet from starting.
+func (ds *dockerService) platformSpecificContainerCreationInitCleanup() (errors []error) {
+	return
+}

pkg/kubelet/dockershim/docker_container_windows.go

@@ -20,10 +20,9 @@ package dockershim
 
 import (
 	"crypto/rand"
-	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
-	"io"
+	"regexp"
 
 	"golang.org/x/sys/windows/registry"
 
@@ -48,7 +47,7 @@ func (ds *dockerService) applyPlatformSpecificDockerConfig(request *runtimeapi.C
 	cleanupInfo := &containerCreationCleanupInfo{}
 
 	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsGMSA) {
-		if err := applyGMSAConfig(request, createConfig, cleanupInfo); err != nil {
+		if err := applyGMSAConfig(request.GetConfig(), createConfig, cleanupInfo); err != nil {
 			return nil, err
 		}
 	}
@@ -63,14 +62,13 @@ func (ds *dockerService) applyPlatformSpecificDockerConfig(request *runtimeapi.C
 // C: drive)
 // When docker supports passing a credential spec's contents directly, we should switch to using that
 // as it will avoid cluttering the registry.
-func applyGMSAConfig(request *runtimeapi.CreateContainerRequest, createConfig *dockertypes.ContainerCreateConfig, cleanupInfo *containerCreationCleanupInfo) error {
+func applyGMSAConfig(config *runtimeapi.ContainerConfig, createConfig *dockertypes.ContainerCreateConfig, cleanupInfo *containerCreationCleanupInfo) error {
-	config := request.GetConfig()
 	credSpec := config.Annotations[kuberuntime.GMSASpecContainerAnnotationKey]
 	if credSpec == "" {
 		return nil
 	}
 
-	valueName, err := copyGMSACredSpecToRegistryValue(credSpec, makeContainerName(request.GetSandboxConfig(), config))
+	valueName, err := copyGMSACredSpecToRegistryValue(credSpec)
 	if err != nil {
 		return err
 	}
@@ -86,15 +84,19 @@ func applyGMSAConfig(request *runtimeapi.CreateContainerRequest, createConfig *d
 }
 
 const (
-	registryNamePrefix = "k8s-cred-spec-"
 	// same as https://github.com/moby/moby/blob/93d994e29c9cc8d81f1b0477e28d705fa7e2cd72/daemon/oci_windows.go#L23
 	credentialSpecRegistryLocation = `SOFTWARE\Microsoft\Windows NT\CurrentVersion\Virtualization\Containers\CredentialSpecs`
+	// the prefix for the registry values we write GMSA cred specs to
+	gMSARegistryValueNamePrefix = "k8s-cred-spec-"
+	// the number of random bytes to generate suffixes for registry value names
+	gMSARegistryValueNameSuffixRandomBytes = 40
 )
 
 // useful to allow mocking the registry in tests
 type registryKey interface {
 	SetStringValue(name, value string) error
 	DeleteValue(name string) error
+	ReadValueNames(n int) ([]string, error)
 	Close() error
 }
 
@@ -106,10 +108,8 @@ var registryCreateKeyFunc = func(baseKey registry.Key, path string, access uint3
 var randomReader = rand.Reader
 
 // copyGMSACredSpecToRegistryKey copies the credential specs to a unique registry value, and returns its name.
-// To avoid leaking registry keys over the life of the node, we generate a unique name for that value, and clean
+func copyGMSACredSpecToRegistryValue(credSpec string) (string, error) {
-// it up after creating the container.
+	valueName, err := gMSARegistryValueName()
-func copyGMSACredSpecToRegistryValue(credSpec string, dockerContainerName string) (string, error) {
-	valueName, err := gMSARegistryValueName(credSpec, dockerContainerName)
 	if err != nil {
 		return "", err
 	}
@@ -128,21 +128,18 @@ func copyGMSACredSpecToRegistryValue(credSpec string, dockerContainerName string
 }
 
 // gMSARegistryValueName computes the name of the registry value where to store the GMSA cred spec contents.
-// The value's name is computed by concatenating the docker container's name (guaranteed to be unique over the
+// The value's name is purely random.
-// container's lifetime), the value itself, and an additional 64 random bytes.
+func gMSARegistryValueName() (string, error) {
-func gMSARegistryValueName(inputs ...string) (string, error) {
+	randBytes := make([]byte, gMSARegistryValueNameSuffixRandomBytes)
-	hasher := sha256.New()
-	for _, s := range inputs {
-		// according to the doc, that can never return an error
-		io.WriteString(hasher, s)
-	}
-	randBytes := make([]byte, 64)
-	if _, err := randomReader.Read(randBytes); err != nil {
-		return "", fmt.Errorf("unable to generate random string: %v", err)
-	}
-	hasher.Write(randBytes)
 
-	return registryNamePrefix + hex.EncodeToString(hasher.Sum(nil)), nil
+	if n, err := randomReader.Read(randBytes); err != nil || n != gMSARegistryValueNameSuffixRandomBytes {
+		if err == nil {
+			err = fmt.Errorf("only got %v random bytes, expected %v", n, len(randBytes))
+		}
+		return "", fmt.Errorf("unable to generate random registry value name: %v", err)
+	}
+
+	return gMSARegistryValueNamePrefix + hex.EncodeToString(randBytes), nil
 }
 
 // performPlatformSpecificContainerCreationCleanup is responsible for doing any platform-specific cleanup
@@ -161,9 +158,8 @@ func (ds *dockerService) performPlatformSpecificContainerCreationCleanup(cleanup
 	return nil
 }
 
-// removeGMSARegistryValue removes the registry value containing the GMSA cred spec for this container, if any.
 func removeGMSARegistryValue(cleanupInfo *containerCreationCleanupInfo) error {
-	if cleanupInfo.gMSARegistryValueName == "" {
+	if cleanupInfo == nil || cleanupInfo.gMSARegistryValueName == "" {
 		return nil
 	}
 
@@ -178,3 +174,39 @@ func removeGMSARegistryValue(cleanupInfo *containerCreationCleanupInfo) error {
 
 	return nil
 }
+
+// platformSpecificContainerCreationKubeletInitCleanup is called when the kubelet
+// is starting, and is meant to clean up any cruft left by previous runs;
+// errors are simply logged, but don't prevent the kubelet from starting.
+func (ds *dockerService) platformSpecificContainerCreationInitCleanup() (errors []error) {
+	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsGMSA) {
+		errors = removeAllGMSARegistryValues()
+	}
+	return
+}
+
+// This is the regex used to detect gMSA cred spec registry values.
+var gMSARegistryValueNamesRegex = regexp.MustCompile(fmt.Sprintf("^%s[0-9a-f]{%d}$", gMSARegistryValueNamePrefix, 2*gMSARegistryValueNameSuffixRandomBytes))
+
+func removeAllGMSARegistryValues() (errors []error) {
+	key, _, err := registryCreateKeyFunc(registry.LOCAL_MACHINE, credentialSpecRegistryLocation, registry.SET_VALUE)
+	if err != nil {
+		return []error{fmt.Errorf("unable to open registry key %s: %v", credentialSpecRegistryLocation, err)}
+	}
+	defer key.Close()
+
+	valueNames, err := key.ReadValueNames(0)
+	if err != nil {
+		return []error{fmt.Errorf("unable to list values under registry key %s: %v", credentialSpecRegistryLocation, err)}
+	}
+
+	for _, valueName := range valueNames {
+		if gMSARegistryValueNamesRegex.MatchString(valueName) {
+			if err = key.DeleteValue(valueName); err != nil {
+				errors = append(errors, fmt.Errorf("unable to remove registry value %s/%s: %v", credentialSpecRegistryLocation, valueName, err))
+			}
+		}
+	}
+
+	return
+}

pkg/kubelet/dockershim/docker_container_windows_test.go

@@ -30,21 +30,35 @@ import (
 )
 
 type dummyRegistryKey struct {
-	setStringError    error
+	setStringValueError error
-	stringValues      [][]string
+	setStringValueArgs  [][]string
-	deleteValueError  error
+
-	deletedValueNames []string
+	deleteValueFunc func(name string) error
-	closed            bool
+	deleteValueArgs []string
+
+	readValueNamesError  error
+	readValueNamesReturn []string
+	readValueNamesArgs   []int
+
+	closed bool
 }
 
 func (k *dummyRegistryKey) SetStringValue(name, value string) error {
-	k.stringValues = append(k.stringValues, []string{name, value})
+	k.setStringValueArgs = append(k.setStringValueArgs, []string{name, value})
-	return k.setStringError
+	return k.setStringValueError
 }
 
 func (k *dummyRegistryKey) DeleteValue(name string) error {
-	k.deletedValueNames = append(k.deletedValueNames, name)
+	k.deleteValueArgs = append(k.deleteValueArgs, name)
-	return k.deleteValueError
+	if k.deleteValueFunc == nil {
+		return nil
+	}
+	return k.deleteValueFunc(name)
+}
+
+func (k *dummyRegistryKey) ReadValueNames(n int) ([]string, error) {
+	k.readValueNamesArgs = append(k.readValueNamesArgs, n)
+	return k.readValueNamesReturn, k.readValueNamesError
 }
 
 func (k *dummyRegistryKey) Close() error {
@@ -54,33 +68,12 @@ func (k *dummyRegistryKey) Close() error {
 
 func TestApplyGMSAConfig(t *testing.T) {
 	dummyCredSpec := "test cred spec contents"
-	randomBytes := []byte{85, 205, 157, 137, 41, 50, 187, 175, 242, 115, 92, 212, 181, 70, 56, 20, 172, 17, 100, 178, 19, 42, 217, 177, 240, 37, 127, 123, 53, 250, 61, 157, 11, 41, 69, 160, 117, 163, 51, 118, 53, 86, 167, 111, 137, 78, 195, 229, 50, 144, 178, 209, 66, 107, 144, 165, 184, 92, 10, 17, 229, 163, 194, 12}
+	randomBytes := []byte{0x19, 0x0, 0x25, 0x45, 0x18, 0x52, 0x9e, 0x2a, 0x3d, 0xed, 0xb8, 0x5c, 0xde, 0xc0, 0x3c, 0xe2, 0x70, 0x55, 0x96, 0x47, 0x45, 0x9a, 0xb5, 0x31, 0xf0, 0x7a, 0xf5, 0xeb, 0x1c, 0x54, 0x95, 0xfd, 0xa7, 0x9, 0x43, 0x5c, 0xe8, 0x2a, 0xb8, 0x9c}
-	expectedHash := "8975ef53024af213c1aca6dfc6e2e48f42c3a984a79e67b140627b8d96007c2a"
+	expectedHex := "1900254518529e2a3dedb85cdec03ce270559647459ab531f07af5eb1c5495fda709435ce82ab89c"
-	expectedValueName := "k8s-cred-spec-" + expectedHash
+	expectedValueName := "k8s-cred-spec-" + expectedHex
 
-	sandboxConfig := &runtimeapi.PodSandboxConfig{
+	containerConfigWithGMSAAnnotation := &runtimeapi.ContainerConfig{
-		Metadata: &runtimeapi.PodSandboxMetadata{
+		Annotations: map[string]string{"container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec},
-			Namespace: "namespace",
-			Uid:       "uid",
-		},
-	}
-
-	containerMeta := &runtimeapi.ContainerMetadata{
-		Name:    "container_name",
-		Attempt: 12,
-	}
-
-	requestWithoutGMSAAnnotation := &runtimeapi.CreateContainerRequest{
-		Config:        &runtimeapi.ContainerConfig{Metadata: containerMeta},
-		SandboxConfig: sandboxConfig,
-	}
-
-	requestWithGMSAAnnotation := &runtimeapi.CreateContainerRequest{
-		Config: &runtimeapi.ContainerConfig{
-			Metadata:    containerMeta,
-			Annotations: map[string]string{"container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec},
-		},
-		SandboxConfig: sandboxConfig,
 	}
 
 	t.Run("happy path", func(t *testing.T) {
@@ -90,17 +83,20 @@ func TestApplyGMSAConfig(t *testing.T) {
 
 		createConfig := &dockertypes.ContainerCreateConfig{}
 		cleanupInfo := &containerCreationCleanupInfo{}
-		err := applyGMSAConfig(requestWithGMSAAnnotation, createConfig, cleanupInfo)
+		err := applyGMSAConfig(containerConfigWithGMSAAnnotation, createConfig, cleanupInfo)
 
 		assert.Nil(t, err)
 
 		// the registry key should have been properly created
-		assert.Equal(t, 1, len(key.stringValues))
+		if assert.Equal(t, 1, len(key.setStringValueArgs)) {
-		assert.Equal(t, []string{expectedValueName, dummyCredSpec}, key.stringValues[0])
+			assert.Equal(t, []string{expectedValueName, dummyCredSpec}, key.setStringValueArgs[0])
+		}
 		assert.True(t, key.closed)
 
 		// the create config's security opt should have been populated
-		assert.Equal(t, createConfig.HostConfig.SecurityOpt, []string{"credentialspec=registry://" + expectedValueName})
+		if assert.NotNil(t, createConfig.HostConfig) {
+			assert.Equal(t, createConfig.HostConfig.SecurityOpt, []string{"credentialspec=registry://" + expectedValueName})
+		}
 
 		// and the name of that value should have been saved to the cleanup info
 		assert.Equal(t, expectedValueName, cleanupInfo.gMSARegistryValueName)
@@ -110,45 +106,58 @@ func TestApplyGMSAConfig(t *testing.T) {
 
 		createConfig := &dockertypes.ContainerCreateConfig{}
 		cleanupInfo := &containerCreationCleanupInfo{}
-		err := applyGMSAConfig(requestWithGMSAAnnotation, createConfig, cleanupInfo)
+		err := applyGMSAConfig(containerConfigWithGMSAAnnotation, createConfig, cleanupInfo)
 
 		assert.Nil(t, err)
 
-		secOpt := createConfig.HostConfig.SecurityOpt[0]
+		if assert.NotNil(t, createConfig.HostConfig) && assert.Equal(t, 1, len(createConfig.HostConfig.SecurityOpt)) {
+			secOpt := createConfig.HostConfig.SecurityOpt[0]
 
-		expectedPrefix := "credentialspec=registry://k8s-cred-spec-"
+			expectedPrefix := "credentialspec=registry://k8s-cred-spec-"
-		assert.Equal(t, expectedPrefix, secOpt[:len(expectedPrefix)])
+			assert.Equal(t, expectedPrefix, secOpt[:len(expectedPrefix)])
 
-		hash := secOpt[len(expectedPrefix):]
+			hex := secOpt[len(expectedPrefix):]
-		hexRegex, _ := regexp.Compile("^[0-9a-f]{64}$")
+			hexRegex := regexp.MustCompile("^[0-9a-f]{80}$")
-		assert.True(t, hexRegex.MatchString(hash))
+			assert.True(t, hexRegex.MatchString(hex))
-		assert.NotEqual(t, expectedHash, hash)
+			assert.NotEqual(t, expectedHex, hex)
 
-		assert.Equal(t, "k8s-cred-spec-"+hash, cleanupInfo.gMSARegistryValueName)
+			assert.Equal(t, "k8s-cred-spec-"+hex, cleanupInfo.gMSARegistryValueName)
+		}
+	})
+	t.Run("when there's an error generating the random value name", func(t *testing.T) {
+		defer setRandomReader([]byte{})()
+
+		err := applyGMSAConfig(containerConfigWithGMSAAnnotation, &dockertypes.ContainerCreateConfig{}, &containerCreationCleanupInfo{})
+
+		if assert.NotNil(t, err) {
+			assert.True(t, strings.Contains(err.Error(), "unable to generate random registry value name"))
+		}
 	})
 	t.Run("if there's an error opening the registry key", func(t *testing.T) {
 		defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{}, fmt.Errorf("dummy error"))()
 
-		err := applyGMSAConfig(requestWithGMSAAnnotation, &dockertypes.ContainerCreateConfig{}, &containerCreationCleanupInfo{})
+		err := applyGMSAConfig(containerConfigWithGMSAAnnotation, &dockertypes.ContainerCreateConfig{}, &containerCreationCleanupInfo{})
 
-		assert.NotNil(t, err)
+		if assert.NotNil(t, err) {
-		assert.True(t, strings.Contains(err.Error(), "unable to open registry key"))
+			assert.True(t, strings.Contains(err.Error(), "unable to open registry key"))
+		}
 	})
-	t.Run("if there's an error writing the registry key", func(t *testing.T) {
+	t.Run("if there's an error writing to the registry key", func(t *testing.T) {
 		key := &dummyRegistryKey{}
-		key.setStringError = fmt.Errorf("dummy error")
+		key.setStringValueError = fmt.Errorf("dummy error")
 		defer setRegistryCreateKeyFunc(t, key)()
 
-		err := applyGMSAConfig(requestWithGMSAAnnotation, &dockertypes.ContainerCreateConfig{}, &containerCreationCleanupInfo{})
+		err := applyGMSAConfig(containerConfigWithGMSAAnnotation, &dockertypes.ContainerCreateConfig{}, &containerCreationCleanupInfo{})
 
-		assert.NotNil(t, err)
+		if assert.NotNil(t, err) {
-		assert.True(t, strings.Contains(err.Error(), "unable to write into registry value"))
+			assert.True(t, strings.Contains(err.Error(), "unable to write into registry value"))
+		}
 		assert.True(t, key.closed)
 	})
 	t.Run("if there is no GMSA annotation", func(t *testing.T) {
 		createConfig := &dockertypes.ContainerCreateConfig{}
 
-		err := applyGMSAConfig(requestWithoutGMSAAnnotation, createConfig, &containerCreationCleanupInfo{})
+		err := applyGMSAConfig(&runtimeapi.ContainerConfig{}, createConfig, &containerCreationCleanupInfo{})
 
 		assert.Nil(t, err)
 		assert.Nil(t, createConfig.HostConfig)
@@ -156,9 +165,7 @@ func TestApplyGMSAConfig(t *testing.T) {
 }
 
 func TestRemoveGMSARegistryValue(t *testing.T) {
-	emptyCleanupInfo := &containerCreationCleanupInfo{}
+	valueName := "k8s-cred-spec-1900254518529e2a3dedb85cdec03ce270559647459ab531f07af5eb1c5495fda709435ce82ab89c"
-
-	valueName := "k8s-cred-spec-8975ef53024af213c1aca6dfc6e2e48f42c3a984a79e67b140627b8d96007c2a"
 	cleanupInfoWithValue := &containerCreationCleanupInfo{gMSARegistryValueName: valueName}
 
 	t.Run("it does remove the registry value", func(t *testing.T) {
@@ -170,8 +177,9 @@ func TestRemoveGMSARegistryValue(t *testing.T) {
 		assert.Nil(t, err)
 
 		// the registry key should have been properly deleted
-		assert.Equal(t, 1, len(key.deletedValueNames))
+		if assert.Equal(t, 1, len(key.deleteValueArgs)) {
-		assert.Equal(t, []string{valueName}, key.deletedValueNames)
+			assert.Equal(t, []string{valueName}, key.deleteValueArgs)
+		}
 		assert.True(t, key.closed)
 	})
 	t.Run("if there's an error opening the registry key", func(t *testing.T) {
@@ -179,24 +187,91 @@ func TestRemoveGMSARegistryValue(t *testing.T) {
 
 		err := removeGMSARegistryValue(cleanupInfoWithValue)
 
-		assert.NotNil(t, err)
+		if assert.NotNil(t, err) {
-		assert.True(t, strings.Contains(err.Error(), "unable to open registry key"))
+			assert.True(t, strings.Contains(err.Error(), "unable to open registry key"))
+		}
 	})
-	t.Run("if there's an error writing the registry key", func(t *testing.T) {
+	t.Run("if there's an error deleting from the registry key", func(t *testing.T) {
 		key := &dummyRegistryKey{}
-		key.deleteValueError = fmt.Errorf("dummy error")
+		key.deleteValueFunc = func(name string) error { return fmt.Errorf("dummy error") }
 		defer setRegistryCreateKeyFunc(t, key)()
 
 		err := removeGMSARegistryValue(cleanupInfoWithValue)
 
-		assert.NotNil(t, err)
+		if assert.NotNil(t, err) {
-		assert.True(t, strings.Contains(err.Error(), "unable to remove registry value"))
+			assert.True(t, strings.Contains(err.Error(), "unable to remove registry value"))
+		}
 		assert.True(t, key.closed)
 	})
-	t.Run("if there's no registry value to be removed", func(t *testing.T) {
+	t.Run("if there's no registry value to be removed, it does nothing", func(t *testing.T) {
-		err := removeGMSARegistryValue(emptyCleanupInfo)
+		key := &dummyRegistryKey{}
+		defer setRegistryCreateKeyFunc(t, key)()
+
+		err := removeGMSARegistryValue(&containerCreationCleanupInfo{})
 
 		assert.Nil(t, err)
+		assert.Equal(t, 0, len(key.deleteValueArgs))
+	})
+}
+
+func TestRemoveAllGMSARegistryValues(t *testing.T) {
+	cred1 := "k8s-cred-spec-1900254518529e2a3dedb85cdec03ce270559647459ab531f07af5eb1c5495fda709435ce82ab89c"
+	cred2 := "k8s-cred-spec-8891436007c795a904fdf77b5348e94305e4c48c5f01c47e7f65e980dc7edda85f112715891d65fd"
+	cred3 := "k8s-cred-spec-2f11f1c9e4f8182fe13caa708bd42b2098c8eefc489d6cc98806c058ccbe4cb3703b9ade61ce59a1"
+	cred4 := "k8s-cred-spec-dc532f189598a8220a1e538f79081eee979f94fbdbf8d37e36959485dee57157c03742d691e1fae2"
+
+	t.Run("it removes the keys matching the k8s creds pattern", func(t *testing.T) {
+		key := &dummyRegistryKey{readValueNamesReturn: []string{cred1, "other_creds", cred2}}
+		defer setRegistryCreateKeyFunc(t, key)()
+
+		errors := removeAllGMSARegistryValues()
+
+		assert.Equal(t, 0, len(errors))
+		assert.Equal(t, []string{cred1, cred2}, key.deleteValueArgs)
+		assert.Equal(t, []int{0}, key.readValueNamesArgs)
+		assert.True(t, key.closed)
+	})
+	t.Run("it ignores errors and does a best effort at removing all k8s creds", func(t *testing.T) {
+		key := &dummyRegistryKey{
+			readValueNamesReturn: []string{cred1, cred2, cred3, cred4},
+			deleteValueFunc: func(name string) error {
+				if name == cred1 || name == cred3 {
+					return fmt.Errorf("dummy error")
+				}
+				return nil
+			},
+		}
+		defer setRegistryCreateKeyFunc(t, key)()
+
+		errors := removeAllGMSARegistryValues()
+
+		assert.Equal(t, 2, len(errors))
+		for _, err := range errors {
+			assert.True(t, strings.Contains(err.Error(), "unable to remove registry value"))
+		}
+		assert.Equal(t, []string{cred1, cred2, cred3, cred4}, key.deleteValueArgs)
+		assert.Equal(t, []int{0}, key.readValueNamesArgs)
+		assert.True(t, key.closed)
+	})
+	t.Run("if there's an error opening the registry key", func(t *testing.T) {
+		defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{}, fmt.Errorf("dummy error"))()
+
+		errors := removeAllGMSARegistryValues()
+
+		if assert.Equal(t, 1, len(errors)) {
+			assert.True(t, strings.Contains(errors[0].Error(), "unable to open registry key"))
+		}
+	})
+	t.Run("if it's unable to list the registry values", func(t *testing.T) {
+		key := &dummyRegistryKey{readValueNamesError: fmt.Errorf("dummy error")}
+		defer setRegistryCreateKeyFunc(t, key)()
+
+		errors := removeAllGMSARegistryValues()
+
+		if assert.Equal(t, 1, len(errors)) {
+			assert.True(t, strings.Contains(errors[0].Error(), "unable to list values under registry key"))
+		}
+		assert.True(t, key.closed)
 	})
 }
 

pkg/kubelet/dockershim/docker_service.go

@@ -394,6 +394,8 @@ func (ds *dockerService) GetPodPortMappings(podSandboxID string) ([]*hostport.Po
 
 // Start initializes and starts components in dockerService.
 func (ds *dockerService) Start() error {
+	ds.initCleanup()
+
 	// Initialize the legacy cleanup flag.
 	if ds.startLocalStreamingServer {
 		go func() {
@@ -405,6 +407,16 @@ func (ds *dockerService) Start() error {
 	return ds.containerManager.Start()
 }
 
+// initCleanup is responsible for cleaning up any crufts left by previous
+// runs. If there are any errros, it simply logs them.
+func (ds *dockerService) initCleanup() {
+	errors := ds.platformSpecificContainerCreationInitCleanup()
+
+	for _, err := range errors {
+		klog.Warningf("initialization error: %v", err)
+	}
+}
+
 // Status returns the status of the runtime.
 func (ds *dockerService) Status(_ context.Context, r *runtimeapi.StatusRequest) (*runtimeapi.StatusResponse, error) {
 	runtimeReady := &runtimeapi.RuntimeCondition{

pkg/kubelet/dockershim/naming.go

@@ -66,7 +66,6 @@ func makeSandboxName(s *runtimeapi.PodSandboxConfig) string {
 	}, nameDelimiter)
 }
 
-// makeContainerName generates a container name that's guaranteed to be unique on its host.
 func makeContainerName(s *runtimeapi.PodSandboxConfig, c *runtimeapi.ContainerConfig) string {
 	return strings.Join([]string{
 		kubePrefix,                            // 0

pkg/kubelet/kuberuntime/kuberuntime_container_windows_test.go

@@ -38,7 +38,7 @@ func TestDetermineEffectiveSecurityContext(t *testing.T) {
 		}
 	}
 
-	t.Run("when there's a specific GMSA for that pod, and no pod-wide GMSA", func(t *testing.T) {
+	t.Run("when there's a specific GMSA for that container, and no pod-wide GMSA", func(t *testing.T) {
 		containerConfig := &runtimeapi.ContainerConfig{}
 
 		pod := buildPod(map[string]string{
@@ -49,7 +49,7 @@ func TestDetermineEffectiveSecurityContext(t *testing.T) {
 
 		assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"])
 	})
-	t.Run("when there's a specific GMSA for that pod, and a pod-wide GMSA", func(t *testing.T) {
+	t.Run("when there's a specific GMSA for that container, and a pod-wide GMSA", func(t *testing.T) {
 		containerConfig := &runtimeapi.ContainerConfig{}
 
 		pod := buildPod(map[string]string{
@@ -61,7 +61,7 @@ func TestDetermineEffectiveSecurityContext(t *testing.T) {
 
 		assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"])
 	})
-	t.Run("when there's no specific GMSA for that pod, and a pod-wide GMSA", func(t *testing.T) {
+	t.Run("when there's no specific GMSA for that container, and a pod-wide GMSA", func(t *testing.T) {
 		containerConfig := &runtimeapi.ContainerConfig{}
 
 		pod := buildPod(map[string]string{
@@ -72,7 +72,7 @@ func TestDetermineEffectiveSecurityContext(t *testing.T) {
 
 		assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"])
 	})
-	t.Run("when there's no specific GMSA for that pod, and no pod-wide GMSA", func(t *testing.T) {
+	t.Run("when there's no specific GMSA for that container, and no pod-wide GMSA", func(t *testing.T) {
 		containerConfig := &runtimeapi.ContainerConfig{}
 
 		determineEffectiveSecurityContext(containerConfig, container, &corev1.Pod{})