Merge pull request #58751 from feiskyer/hyperv

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add support of hyperv isolation for windows containers

**What this PR does / why we need it**:

Add support of hyperv isolation for windows containers.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #58750

**Special notes for your reviewer**:

Only one container per pod is supported yet.

**Release note**:

```release-note
Windows containers now support experimental Hyper-V isolation by setting annotation `experimental.windows.kubernetes.io/isolation-type=hyperv` and feature gates HyperVContainer. Only one container per pod is supported yet.
```

pkg/features/kube_features.go

@@ -225,6 +225,12 @@ const (
 	//
 	// Implement support for limiting pids in pods
 	SupportPodPidsLimit utilfeature.Feature = "SupportPodPidsLimit"
+
+	// owner: @feiskyer
+	// alpha: v1.10
+	//
+	// Enable Hyper-V containers on Windows
+	HyperVContainer utilfeature.Feature = "HyperVContainer"
 )
 
 func init() {
@@ -265,6 +271,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	ResourceLimitsPriorityFunction:              {Default: false, PreRelease: utilfeature.Alpha},
 	SupportIPVSProxyMode:                        {Default: false, PreRelease: utilfeature.Beta},
 	SupportPodPidsLimit:                         {Default: false, PreRelease: utilfeature.Alpha},
+	HyperVContainer:                             {Default: false, PreRelease: utilfeature.Alpha},
 
 	// inherited features from generic apiserver, relisted here to get a conflict if it is changed
 	// unintentionally on either side:

pkg/kubelet/dockershim/BUILD

@@ -124,7 +124,13 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//vendor/k8s.io/client-go/tools/remotecommand:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
-    ],
+    ] + select({
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//pkg/features:go_default_library",
+            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 go_test(

pkg/kubelet/dockershim/docker_sandbox.go

@@ -587,6 +587,8 @@ func (ds *dockerService) makeSandboxDockerConfig(c *runtimeapi.PodSandboxConfig,
 		return nil, fmt.Errorf("failed to generate sandbox security options for sandbox %q: %v", c.Metadata.Name, err)
 	}
 	hc.SecurityOpt = append(hc.SecurityOpt, securityOpts...)
+
+	applyExperimentalCreateConfig(createConfig, c.Annotations)
 	return createConfig, nil
 }
 

pkg/kubelet/dockershim/helpers_linux.go

@@ -145,3 +145,7 @@ func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) {
 	}
 	return fmt.Sprintf(dockerNetNSFmt, c.State.Pid), nil
 }
+
+// applyExperimentalCreateConfig applys experimental configures from sandbox annotations.
+func applyExperimentalCreateConfig(createConfig *dockertypes.ContainerCreateConfig, annotations map[string]string) {
+}

pkg/kubelet/dockershim/helpers_unsupported.go

@@ -53,3 +53,7 @@ func (ds *dockerService) determinePodIPBySandboxID(uid string) string {
 func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) {
 	return "", fmt.Errorf("unsupported platform")
 }
+
+// applyExperimentalCreateConfig applys experimental configures from sandbox annotations.
+func applyExperimentalCreateConfig(createConfig *dockertypes.ContainerCreateConfig, annotations map[string]string) {
+}

pkg/kubelet/dockershim/helpers_windows.go

@@ -26,9 +26,19 @@ import (
 	dockercontainer "github.com/docker/docker/api/types/container"
 	dockerfilters "github.com/docker/docker/api/types/filters"
 	"github.com/golang/glog"
+
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/kubernetes/pkg/features"
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 )
 
+const (
+	hypervIsolationAnnotationKey = "experimental.windows.kubernetes.io/isolation-type"
+
+	// Refer https://aka.ms/hyperv-container.
+	hypervIsolation = "hyperv"
+)
+
 func DefaultMemorySwap() int64 {
 	return 0
 }
@@ -40,6 +50,26 @@ func (ds *dockerService) getSecurityOpts(seccompProfile string, separator rune)
 	return nil, nil
 }
 
+func shouldIsolatedByHyperV(annotations map[string]string) bool {
+	if !utilfeature.DefaultFeatureGate.Enabled(features.HyperVContainer) {
+		return false
+	}
+
+	v, ok := annotations[hypervIsolationAnnotationKey]
+	return ok && v == hypervIsolation
+}
+
+// applyExperimentalCreateConfig applys experimental configures from sandbox annotations.
+func applyExperimentalCreateConfig(createConfig *dockertypes.ContainerCreateConfig, annotations map[string]string) {
+	if shouldIsolatedByHyperV(annotations) {
+		createConfig.HostConfig.Isolation = hypervIsolation
+
+		if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode == "" {
+			createConfig.HostConfig.NetworkMode = dockercontainer.NetworkMode("none")
+		}
+	}
+}
+
 func (ds *dockerService) updateCreateConfig(
 	createConfig *dockertypes.ContainerCreateConfig,
 	config *runtimeapi.ContainerConfig,
@@ -47,11 +77,13 @@ func (ds *dockerService) updateCreateConfig(
 	podSandboxID string, securityOptSep rune, apiVersion *semver.Version) error {
 	if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode != "" {
 		createConfig.HostConfig.NetworkMode = dockercontainer.NetworkMode(networkMode)
-	} else {
+	} else if !shouldIsolatedByHyperV(sandboxConfig.Annotations) {
 		// Todo: Refactor this call in future for calling methods directly in security_context.go
 		modifyHostNetworkOptionForContainer(false, podSandboxID, createConfig.HostConfig)
 	}
 
+	applyExperimentalCreateConfig(createConfig, sandboxConfig.Annotations)
+
 	return nil
 }
 
@@ -87,8 +119,17 @@ func (ds *dockerService) determinePodIPBySandboxID(sandboxID string) string {
 		// Todo: Add a kernel version check for more validation
 
 		if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode == "" {
-			// Do not return any IP, so that we would continue and get the IP of the Sandbox
+			if r.HostConfig.Isolation == hypervIsolation {
-			ds.getIP(sandboxID, r)
+				// Hyper-V only supports one container per Pod yet and the container will have a different
+				// IP address from sandbox. Return the first non-sandbox container IP as POD IP.
+				// TODO(feiskyer): remove this workaround after Hyper-V supports multiple containers per Pod.
+				if containerIP := ds.getIP(c.ID, r); containerIP != "" {
+					return containerIP
+				}
+			} else {
+				// Do not return any IP, so that we would continue and get the IP of the Sandbox
+				ds.getIP(sandboxID, r)
+			}
 		} else {
 			// On Windows, every container that is created in a Sandbox, needs to invoke CNI plugin again for adding the Network,
 			// with the shared container name as NetNS info,