Merge pull request #97141 from wawa0210/remove-hyperv

remove experimental windows container hyper-v support with Docker

pkg/features/kube_features.go

@@ -183,13 +183,6 @@ const (
 	// Implement support for limiting pids in pods
 	SupportPodPidsLimit featuregate.Feature = "SupportPodPidsLimit"
 
-	// owner: @feiskyer
-	// alpha: v1.10
-	//
-	// Enable Hyper-V containers on Windows
-	// Deprecated in 1.20 and removed in 1.21
-	HyperVContainer featuregate.Feature = "HyperVContainer"
-
 	// owner: @mikedanese
 	// alpha: v1.13
 	//
@@ -709,7 +702,6 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 	StorageObjectInUseProtection:   {Default: true, PreRelease: featuregate.GA},
 	SupportPodPidsLimit:            {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23
 	SupportNodePidsLimit:           {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23
-	HyperVContainer:                {Default: false, PreRelease: featuregate.Deprecated},
 	BoundServiceAccountTokenVolume: {Default: false, PreRelease: featuregate.Alpha},
 	ServiceAccountIssuerDiscovery:  {Default: true, PreRelease: featuregate.Beta},
 	CRIContainerLogRotation:        {Default: true, PreRelease: featuregate.Beta},

pkg/kubelet/apis/BUILD

@@ -7,22 +7,12 @@ load(
 
 go_library(
     name = "go_default_library",
-    srcs = [
+    srcs = ["well_known_labels.go"],
-        "well_known_annotations_windows.go",
-        "well_known_labels.go",
-    ],
     importpath = "k8s.io/kubernetes/pkg/kubelet/apis",
     deps = [
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
-    ] + select({
-        "@io_bazel_rules_go//go/platform:windows": [
-            "//pkg/features:go_default_library",
-            "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
-            "//vendor/k8s.io/klog/v2:go_default_library",
     ],
-        "//conditions:default": [],
-    }),
 )
 
 filegroup(

pkg/kubelet/apis/well_known_annotations_windows.go

@@ -1,46 +0,0 @@
-// +build windows
-
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package apis
-
-import (
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	"k8s.io/klog/v2"
-	"k8s.io/kubernetes/pkg/features"
-)
-
-const (
-	// HypervIsolationAnnotationKey is used to run windows containers with hyperv isolation.
-	// Refer https://aka.ms/hyperv-container.
-	HypervIsolationAnnotationKey = "experimental.windows.kubernetes.io/isolation-type"
-	// HypervIsolationValue is used to run windows containers with hyperv isolation.
-	// Refer https://aka.ms/hyperv-container.
-	HypervIsolationValue = "hyperv"
-)
-
-// ShouldIsolatedByHyperV returns true if a windows container should be run with hyperv isolation.
-func ShouldIsolatedByHyperV(annotations map[string]string) bool {
-	klog.Warningf("The hyper-v FeatureGate is deprecated in 1.20 and will be removed in 1.21")
-
-	if !utilfeature.DefaultFeatureGate.Enabled(features.HyperVContainer) {
-		return false
-	}
-
-	v, ok := annotations[HypervIsolationAnnotationKey]
-	return ok && v == HypervIsolationValue
-}

pkg/kubelet/dockershim/BUILD

@@ -76,7 +76,6 @@ go_library(
         "//vendor/k8s.io/utils/exec:go_default_library",
     ] + select({
         "@io_bazel_rules_go//go/platform:windows": [
-            "//pkg/kubelet/apis:go_default_library",
             "//pkg/kubelet/winstats:go_default_library",
             "//vendor/github.com/Microsoft/hcsshim:go_default_library",
             "//vendor/golang.org/x/sys/windows/registry:go_default_library",

pkg/kubelet/dockershim/docker_sandbox.go

@@ -662,7 +662,6 @@ func (ds *dockerService) makeSandboxDockerConfig(c *runtimeapi.PodSandboxConfig,
 	securityOpts := ds.getSandBoxSecurityOpts(securityOptSeparator)
 	hc.SecurityOpt = append(hc.SecurityOpt, securityOpts...)
 
-	applyExperimentalCreateConfig(createConfig, c.Annotations)
 	return createConfig, nil
 }
 

pkg/kubelet/dockershim/helpers_windows.go

@@ -29,7 +29,6 @@ import (
 	"k8s.io/klog/v2"
 
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
-	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 )
 
 // DefaultMemorySwap always returns 0 for no memory swap in a sandbox
@@ -50,17 +49,6 @@ func (ds *dockerService) getSandBoxSecurityOpts(separator rune) []string {
 	return nil
 }
 
-// applyExperimentalCreateConfig applys experimental configures from sandbox annotations.
-func applyExperimentalCreateConfig(createConfig *dockertypes.ContainerCreateConfig, annotations map[string]string) {
-	if kubeletapis.ShouldIsolatedByHyperV(annotations) {
-		createConfig.HostConfig.Isolation = kubeletapis.HypervIsolationValue
-
-		if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode == "" {
-			createConfig.HostConfig.NetworkMode = dockercontainer.NetworkMode("none")
-		}
-	}
-}
-
 func (ds *dockerService) updateCreateConfig(
 	createConfig *dockertypes.ContainerCreateConfig,
 	config *runtimeapi.ContainerConfig,
@@ -68,7 +56,7 @@ func (ds *dockerService) updateCreateConfig(
 	podSandboxID string, securityOptSep rune, apiVersion *semver.Version) error {
 	if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode != "" {
 		createConfig.HostConfig.NetworkMode = dockercontainer.NetworkMode(networkMode)
-	} else if !kubeletapis.ShouldIsolatedByHyperV(sandboxConfig.Annotations) {
+	} else {
 		// Todo: Refactor this call in future for calling methods directly in security_context.go
 		modifyHostOptionsForContainer(nil, podSandboxID, createConfig.HostConfig)
 	}
@@ -90,8 +78,6 @@ func (ds *dockerService) updateCreateConfig(
 		applyWindowsContainerSecurityContext(wc.GetSecurityContext(), createConfig.Config, createConfig.HostConfig)
 	}
 
-	applyExperimentalCreateConfig(createConfig, sandboxConfig.Annotations)
-
 	return nil
 }
 
@@ -149,21 +135,12 @@ func (ds *dockerService) determinePodIPBySandboxID(sandboxID string) []string {
 			// Instead of relying on this call, an explicit call to addToNetwork should be
 			// done immediately after ContainerCreation, in case of Windows only. TBD Issue # to handle this
 
-			if r.HostConfig.Isolation == kubeletapis.HypervIsolationValue {
-				// Hyper-V only supports one container per Pod yet and the container will have a different
-				// IP address from sandbox. Return the first non-sandbox container IP as POD IP.
-				// TODO(feiskyer): remove this workaround after Hyper-V supports multiple containers per Pod.
-				if containerIPs := ds.getIPs(c.ID, r); len(containerIPs) != 0 {
-					return containerIPs
-				}
-			} else {
 			// Do not return any IP, so that we would continue and get the IP of the Sandbox.
 			// Windows 1709 and 1803 doesn't have the Namespace support, so getIP() is called
 			// to replicate the DNS registry key to the Workload container (IP/Gateway/MAC is
 			// set separately than DNS).
 			// TODO(feiskyer): remove this workaround after Namespace is supported in Windows RS5.
 			ds.getIPs(sandboxID, r)
-			}
 		} else {
 			// ds.getIP will call the CNI plugin to fetch the IP
 			if containerIPs := ds.getIPs(c.ID, r); len(containerIPs) != 0 {

pkg/kubelet/kuberuntime/BUILD

@@ -87,9 +87,6 @@ go_library(
             "//pkg/kubelet/qos:go_default_library",
             "//vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs:go_default_library",
         ],
-        "@io_bazel_rules_go//go/platform:windows": [
-            "//pkg/kubelet/apis:go_default_library",
-        ],
         "//conditions:default": [],
     }),
 )

pkg/kubelet/kuberuntime/kuberuntime_container_windows.go

@@ -25,7 +25,6 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 	kubefeatures "k8s.io/kubernetes/pkg/features"
-	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/securitycontext"
 
@@ -52,7 +51,6 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1
 	}
 
 	cpuLimit := container.Resources.Limits.Cpu()
-	isolatedByHyperv := kubeletapis.ShouldIsolatedByHyperV(pod.Annotations)
 	if !cpuLimit.IsZero() {
 		// Note that sysinfo.NumCPU() is limited to 64 CPUs on Windows due to Processor Groups,
 		// as only 64 processors are available for execution by a given process. This causes
@@ -85,16 +83,6 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1
 
 		cpuMaximum := 10000 * cpuLimit.MilliValue() / int64(runtime.NumCPU()) / 1000
 
-		// TODO: This should be reviewed or removed once Hyper-V support is implemented with CRI-ContainerD
-		//       in a future release. cpuCount may or may not be required if cpuMaximum is set.
-		if isolatedByHyperv {
-			cpuCount := int64(cpuLimit.MilliValue()+999) / 1000
-			wc.Resources.CpuCount = cpuCount
-
-			if cpuCount != 0 {
-				cpuMaximum = cpuLimit.MilliValue() / cpuCount * 10000 / 1000
-			}
-		}
 		// ensure cpuMaximum is in range [1, 10000].
 		if cpuMaximum < 1 {
 			cpuMaximum = 1
@@ -105,7 +93,6 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1
 		wc.Resources.CpuMaximum = cpuMaximum
 	}
 
-	if !isolatedByHyperv {
 	// The processor resource controls are mutually exclusive on
 	// Windows Server Containers, the order of precedence is
 	// CPUCount first, then CPUMaximum.
@@ -115,7 +102,6 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1
 			klog.Warningf("Mutually exclusive options: CPUCount priority > CPUMaximum priority on Windows Server Containers. CPUMaximum should be ignored")
 		}
 	}
-	}
 
 	memoryLimit := container.Resources.Limits.Memory().Value()
 	if memoryLimit != 0 {